2024-11-11 23:52:40,459 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-11 23:52:40,471 main DEBUG Took 0.009952 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 23:52:40,471 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 23:52:40,471 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 23:52:40,472 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 23:52:40,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,482 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 23:52:40,492 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,494 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,495 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,496 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,497 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,498 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,499 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,500 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,501 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,502 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,503 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:52:40,504 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,504 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 23:52:40,505 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:52:40,506 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 23:52:40,508 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 23:52:40,509 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 23:52:40,510 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 23:52:40,510 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 23:52:40,518 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 23:52:40,520 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 23:52:40,522 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 23:52:40,522 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 23:52:40,523 main DEBUG createAppenders(={Console}) 2024-11-11 23:52:40,524 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-11 23:52:40,524 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-11 23:52:40,524 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-11 23:52:40,525 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 23:52:40,525 main DEBUG OutputStream closed 2024-11-11 23:52:40,525 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 23:52:40,526 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 23:52:40,526 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2dc54ad4 OK 2024-11-11 23:52:40,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 23:52:40,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 23:52:40,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 23:52:40,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 23:52:40,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 23:52:40,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 23:52:40,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 23:52:40,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 23:52:40,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 23:52:40,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 23:52:40,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 23:52:40,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 23:52:40,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 23:52:40,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 23:52:40,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 23:52:40,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 23:52:40,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 23:52:40,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 23:52:40,618 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 23:52:40,618 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4f9a2c08) with optional ClassLoader: null 2024-11-11 23:52:40,619 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 23:52:40,619 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4f9a2c08] started OK. 2024-11-11T23:52:40,843 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6 2024-11-11 23:52:40,845 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 23:52:40,846 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T23:52:40,880 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStreamFSHLog timeout: 26 mins 2024-11-11T23:52:40,908 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d, deleteOnExit=true 2024-11-11T23:52:40,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/test.cache.data in system properties and HBase conf 2024-11-11T23:52:40,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T23:52:40,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir in system properties and HBase conf 2024-11-11T23:52:40,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T23:52:40,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T23:52:40,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T23:52:41,016 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T23:52:41,130 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T23:52:41,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:52:41,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:52:41,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T23:52:41,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:52:41,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T23:52:41,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T23:52:41,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:52:41,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:52:41,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T23:52:41,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/nfs.dump.dir in system properties and HBase conf 2024-11-11T23:52:41,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/java.io.tmpdir in system properties and HBase conf 2024-11-11T23:52:41,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:52:41,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T23:52:41,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T23:52:42,075 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T23:52:42,189 INFO [Time-limited test {}] log.Log(170): Logging initialized @2405ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T23:52:42,285 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:52:42,361 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:52:42,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:52:42,406 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:52:42,408 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T23:52:42,425 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:52:42,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:52:42,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:52:42,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44567ea4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/java.io.tmpdir/jetty-localhost-34629-hadoop-hdfs-3_4_1-tests_jar-_-any-11856974237965408858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:52:42,671 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:34629} 2024-11-11T23:52:42,671 INFO [Time-limited test {}] server.Server(415): Started @2889ms 2024-11-11T23:52:43,105 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:52:43,115 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:52:43,116 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:52:43,117 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:52:43,117 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T23:52:43,118 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:52:43,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:52:43,227 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a86aa33{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/java.io.tmpdir/jetty-localhost-39141-hadoop-hdfs-3_4_1-tests_jar-_-any-10557415625718852425/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:43,228 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:39141} 2024-11-11T23:52:43,229 INFO [Time-limited test {}] server.Server(415): Started @3446ms 2024-11-11T23:52:43,285 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T23:52:43,435 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:52:43,442 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:52:43,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:52:43,451 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:52:43,451 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T23:52:43,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:52:43,453 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:52:43,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fecc8a3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/java.io.tmpdir/jetty-localhost-44759-hadoop-hdfs-3_4_1-tests_jar-_-any-17236390873985446102/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:43,601 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:44759} 2024-11-11T23:52:43,602 INFO [Time-limited test {}] server.Server(415): Started @3819ms 2024-11-11T23:52:43,605 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T23:52:43,674 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:52:43,682 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:52:43,688 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data1/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,688 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data2/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:52:43,690 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:52:43,690 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T23:52:43,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:52:43,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:52:43,720 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data3/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,725 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data4/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,750 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T23:52:43,753 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T23:52:43,819 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe7fde3932ada004e with lease ID 0xc50f80046d5560eb: Processing first storage report for DS-19d5e64d-6cdd-419f-a545-2f0f55f72517 from datanode DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe7fde3932ada004e with lease ID 0xc50f80046d5560eb: from storage DS-19d5e64d-6cdd-419f-a545-2f0f55f72517 node DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T23:52:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf36e92654726b6e8 with lease ID 0xc50f80046d5560ea: Processing first storage report for DS-2a3ad448-ec73-4af9-84f3-80d9bcf188cd from datanode DatanodeRegistration(127.0.0.1:37561, datanodeUuid=70be881c-ee35-4e55-8d9a-bf699cd3174a, infoPort=38179, infoSecurePort=0, ipcPort=33947, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf36e92654726b6e8 with lease ID 0xc50f80046d5560ea: from storage DS-2a3ad448-ec73-4af9-84f3-80d9bcf188cd node DatanodeRegistration(127.0.0.1:37561, datanodeUuid=70be881c-ee35-4e55-8d9a-bf699cd3174a, infoPort=38179, infoSecurePort=0, ipcPort=33947, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T23:52:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf36e92654726b6e8 with lease ID 0xc50f80046d5560ea: Processing first storage report for DS-2aa94765-39dd-44f4-8805-09876237c45a from datanode DatanodeRegistration(127.0.0.1:37561, datanodeUuid=70be881c-ee35-4e55-8d9a-bf699cd3174a, infoPort=38179, infoSecurePort=0, ipcPort=33947, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf36e92654726b6e8 with lease ID 0xc50f80046d5560ea: from storage DS-2aa94765-39dd-44f4-8805-09876237c45a node DatanodeRegistration(127.0.0.1:37561, datanodeUuid=70be881c-ee35-4e55-8d9a-bf699cd3174a, infoPort=38179, infoSecurePort=0, ipcPort=33947, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T23:52:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe7fde3932ada004e with lease ID 0xc50f80046d5560eb: Processing first storage report for DS-54e5c060-a8ca-44e5-b551-8205ab76d9ed from datanode DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe7fde3932ada004e with lease ID 0xc50f80046d5560eb: from storage DS-54e5c060-a8ca-44e5-b551-8205ab76d9ed node DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T23:52:43,836 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d954ade{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/java.io.tmpdir/jetty-localhost-45491-hadoop-hdfs-3_4_1-tests_jar-_-any-13754706935361017942/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:43,837 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:45491} 2024-11-11T23:52:43,837 INFO [Time-limited test {}] server.Server(415): Started @4054ms 2024-11-11T23:52:43,839 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T23:52:43,936 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data5/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,941 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data6/current/BP-740878948-172.17.0.2-1731369161785/current, will proceed with Du for space computation calculation, 2024-11-11T23:52:43,966 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T23:52:43,977 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf8f0b30bfce7382e with lease ID 0xc50f80046d5560ec: Processing first storage report for DS-d669d299-e12c-4379-99c9-5201e5ba314d from datanode DatanodeRegistration(127.0.0.1:44147, datanodeUuid=1e8c5503-00ac-424f-a1d0-54cf5db073a2, infoPort=38515, infoSecurePort=0, ipcPort=40457, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,977 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf8f0b30bfce7382e with lease ID 0xc50f80046d5560ec: from storage DS-d669d299-e12c-4379-99c9-5201e5ba314d node DatanodeRegistration(127.0.0.1:44147, datanodeUuid=1e8c5503-00ac-424f-a1d0-54cf5db073a2, infoPort=38515, infoSecurePort=0, ipcPort=40457, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T23:52:43,978 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf8f0b30bfce7382e with lease ID 0xc50f80046d5560ec: Processing first storage report for DS-0bc5eb83-d979-4669-9912-493e0e342163 from datanode DatanodeRegistration(127.0.0.1:44147, datanodeUuid=1e8c5503-00ac-424f-a1d0-54cf5db073a2, infoPort=38515, infoSecurePort=0, ipcPort=40457, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785) 2024-11-11T23:52:43,978 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf8f0b30bfce7382e with lease ID 0xc50f80046d5560ec: from storage DS-0bc5eb83-d979-4669-9912-493e0e342163 node DatanodeRegistration(127.0.0.1:44147, datanodeUuid=1e8c5503-00ac-424f-a1d0-54cf5db073a2, infoPort=38515, infoSecurePort=0, ipcPort=40457, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T23:52:44,205 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6 2024-11-11T23:52:44,216 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=154, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5829 2024-11-11T23:52:44,218 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T23:52:44,869 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T23:52:44,890 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:44,966 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:45,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369165008 2024-11-11T23:52:45,234 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369165008 2024-11-11T23:52:45,293 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:45,373 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369165008, startPosition=0, beingWritten=true 2024-11-11T23:52:45,521 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369165008 to pos 203, reset compression=false 2024-11-11T23:52:45,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:45,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:45,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:45,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:45,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741825_1001 (size=447) 2024-11-11T23:52:45,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741825_1001 (size=447) 2024-11-11T23:52:45,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741825_1001 (size=447) 2024-11-11T23:52:46,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:46,028 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369165008) 2024-11-11T23:52:46,037 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=165 (was 154) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@88a5362 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38259 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=411 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5600 (was 5829) 2024-11-11T23:52:46,045 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=165, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5598 2024-11-11T23:52:46,047 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:46,054 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:46,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 2024-11-11T23:52:46,067 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 2024-11-11T23:52:46,070 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:46,074 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056, startPosition=0, beingWritten=true 2024-11-11T23:52:46,161 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:46,164 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056, startPosition=0, beingWritten=true 2024-11-11T23:52:46,176 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:46,200 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056, startPosition=0, beingWritten=true 2024-11-11T23:52:46,211 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:46,243 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056, startPosition=0, beingWritten=true 2024-11-11T23:52:46,259 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:46,300 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056, startPosition=0, beingWritten=true 2024-11-11T23:52:46,316 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:46,317 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 to pos 460, reset compression=false 2024-11-11T23:52:46,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741826_1002 (size=468) 2024-11-11T23:52:46,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741826_1002 (size=468) 2024-11-11T23:52:46,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741826_1002 (size=468) 2024-11-11T23:52:46,343 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:46,343 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369166056) 2024-11-11T23:52:46,345 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 to pos 460, reset compression=false 2024-11-11T23:52:46,354 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=171 (was 165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-175 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=411 (was 411), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5584 (was 5598) 2024-11-11T23:52:46,355 INFO [Thread-175 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 2024-11-11T23:52:46,363 DEBUG [Thread-175 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369166056 2024-11-11T23:52:46,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=171, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=12, AvailableMemoryMB=5582 2024-11-11T23:52:46,364 DEBUG [Thread-175 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:46,364 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:46,364 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:46,371 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:46,374 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731369166374 2024-11-11T23:52:46,390 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731369166374 2024-11-11T23:52:46,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:46,513 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.false 2024-11-11T23:52:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741828_1004 (size=231) 2024-11-11T23:52:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741828_1004 (size=231) 2024-11-11T23:52:46,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741828_1004 (size=231) 2024-11-11T23:52:46,538 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-11T23:52:46,539 DEBUG [Thread-198 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/user/jenkins/log.1.false, startPosition=0, beingWritten=false 2024-11-11T23:52:46,546 DEBUG [Thread-198 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/user/jenkins/log.1.false 2024-11-11T23:52:46,547 DEBUG [Thread-198 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:46,547 DEBUG [Thread-198 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/user/jenkins/log.2.false, startPosition=0, beingWritten=false 2024-11-11T23:52:46,550 WARN [Thread-198 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38259/user/jenkins/log.2.false, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:46,550 DEBUG [Thread-198 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/user/jenkins/log.2.false 2024-11-11T23:52:46,551 DEBUG [Thread-198 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:46,551 DEBUG [Thread-198 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:46,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741827_1003 (size=93) 2024-11-11T23:52:46,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741827_1003 (size=93) 2024-11-11T23:52:46,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741827_1003 (size=93) 2024-11-11T23:52:46,568 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:46,568 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369166374) 2024-11-11T23:52:46,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=176 (was 171) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=423 (was 411) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 12), AvailableMemoryMB=5559 (was 5582) 2024-11-11T23:52:46,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=176, OpenFileDescriptor=423, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5557 2024-11-11T23:52:46,590 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:46,596 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:46,600 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,616 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,621 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:46,623 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 2024-11-11T23:52:46,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,635 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 2024-11-11T23:52:46,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:46,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 is not closed yet, will try archiving it next time 2024-11-11T23:52:46,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741829_1005 (size=93) 2024-11-11T23:52:46,639 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:46,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741829_1005 (size=93) 2024-11-11T23:52:46,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741829_1005 (size=93) 2024-11-11T23:52:46,641 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,741 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,755 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:46,756 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:46,756 DEBUG [Thread-215 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599, startPosition=0, beingWritten=false 2024-11-11T23:52:46,759 INFO [Thread-215 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,761 WARN [Thread-215 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:46,762 DEBUG [Thread-215 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166599 2024-11-11T23:52:46,762 DEBUG [Thread-215 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623, startPosition=0, beingWritten=true 2024-11-11T23:52:46,796 DEBUG [Thread-215 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 to pos 0, reset compression=false 2024-11-11T23:52:46,817 DEBUG [Thread-215 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 to pos 0, reset compression=false 2024-11-11T23:52:46,841 DEBUG [Thread-215 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 to pos 0, reset compression=false 2024-11-11T23:52:46,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,859 DEBUG [Thread-215 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 to pos 0, reset compression=false 2024-11-11T23:52:46,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741830_1006 (size=93) 2024-11-11T23:52:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741830_1006 (size=93) 2024-11-11T23:52:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741830_1006 (size=93) 2024-11-11T23:52:46,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:46,867 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369166623) 2024-11-11T23:52:46,870 DEBUG [Thread-215 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731369166623 2024-11-11T23:52:46,871 DEBUG [Thread-215 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:46,871 DEBUG [Thread-215 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:46,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=183 (was 176) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #7] java.base@17.0.11/java.io.RandomAccessFile.open0(Native Method) java.base@17.0.11/java.io.RandomAccessFile.open(RandomAccessFile.java:344) java.base@17.0.11/java.io.RandomAccessFile.(RandomAccessFile.java:259) java.base@17.0.11/java.io.RandomAccessFile.(RandomAccessFile.java:213) app//org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil.openAndSeek(FsDatasetUtil.java:116) app//org.apache.hadoop.hdfs.server.datanode.FileIoProvider.openAndSeek(FileIoProvider.java:463) app//org.apache.hadoop.hdfs.server.datanode.LocalReplica.getMetadataInputStream(LocalReplica.java:295) app//org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:250) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-215 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:419) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) app//org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) app//org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hbase.io.util.StreamUtils.readRawVarint32(StreamUtils.java:54) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.skipHeader(ProtobufWALTailingReader.java:265) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:286) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=425 (was 423) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5521 (was 5557) 2024-11-11T23:52:46,885 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=183, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5520 2024-11-11T23:52:46,886 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:46,891 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:46,892 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892 2024-11-11T23:52:46,901 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892 2024-11-11T23:52:46,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:46,908 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166907 2024-11-11T23:52:46,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,920 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892 with entries=10, filesize=1.30 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166907 2024-11-11T23:52:46,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:46,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892 is not closed yet, will try archiving it next time 2024-11-11T23:52:46,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741831_1007 (size=1343) 2024-11-11T23:52:46,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741831_1007 (size=1343) 2024-11-11T23:52:46,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741831_1007 (size=1343) 2024-11-11T23:52:46,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741832_1008 (size=718) 2024-11-11T23:52:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741832_1008 (size=718) 2024-11-11T23:52:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741832_1008 (size=718) 2024-11-11T23:52:46,934 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-11T23:52:46,935 DEBUG [Thread-238 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892, startPosition=0, beingWritten=false 2024-11-11T23:52:46,945 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-11T23:52:46,946 DEBUG [Thread-238 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166892 2024-11-11T23:52:46,946 DEBUG [Thread-238 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166907, startPosition=0, beingWritten=false 2024-11-11T23:52:46,954 DEBUG [Thread-238 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731369166907 2024-11-11T23:52:46,954 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-11T23:52:46,954 DEBUG [Thread-238 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:46,955 DEBUG [Thread-238 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:46,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:46,959 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369166907) 2024-11-11T23:52:46,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=188 (was 183) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5516 (was 5520) 2024-11-11T23:52:46,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=188, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5515 2024-11-11T23:52:46,974 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:46,979 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:46,981 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369166980 2024-11-11T23:52:46,995 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369166980 2024-11-11T23:52:46,998 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:47,016 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-11T23:52:47,018 DEBUG [Thread-248 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-11T23:52:47,020 WARN [Thread-248 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:47,020 DEBUG [Thread-248 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-11T23:52:47,021 DEBUG [Thread-248 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:47,021 DEBUG [Thread-248 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:47,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741833_1009 (size=93) 2024-11-11T23:52:47,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741833_1009 (size=93) 2024-11-11T23:52:47,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741833_1009 (size=93) 2024-11-11T23:52:47,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369166980) 2024-11-11T23:52:47,043 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=193 (was 188) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5513 (was 5515) 2024-11-11T23:52:47,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=193, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5513 2024-11-11T23:52:47,051 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,055 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057 2024-11-11T23:52:47,068 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057 2024-11-11T23:52:47,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:47,072 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 2024-11-11T23:52:47,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,082 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 2024-11-11T23:52:47,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:47,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057 is not closed yet, will try archiving it next time 2024-11-11T23:52:47,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741834_1010 (size=218) 2024-11-11T23:52:47,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741834_1010 (size=218) 2024-11-11T23:52:47,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741834_1010 (size=218) 2024-11-11T23:52:47,089 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,094 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:47,095 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057, startPosition=0, beingWritten=false 2024-11-11T23:52:47,101 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167057 2024-11-11T23:52:47,102 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-11T23:52:47,102 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071, startPosition=0, beingWritten=true 2024-11-11T23:52:47,116 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-11T23:52:47,116 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 to pos 2585, reset compression=false 2024-11-11T23:52:47,117 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167116 2024-11-11T23:52:47,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,128 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 with entries=20, filesize=2.52 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167116 2024-11-11T23:52:47,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:47,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 is not closed yet, will try archiving it next time 2024-11-11T23:52:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741835_1011 (size=2593) 2024-11-11T23:52:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741835_1011 (size=2593) 2024-11-11T23:52:47,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741835_1011 (size=2593) 2024-11-11T23:52:47,150 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 to pos 2585, reset compression=false 2024-11-11T23:52:47,154 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167071 2024-11-11T23:52:47,154 DEBUG [Thread-265 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167116, startPosition=0, beingWritten=true 2024-11-11T23:52:47,167 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-11T23:52:47,168 DEBUG [Thread-265 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167116 to pos 1335, reset compression=false 2024-11-11T23:52:47,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741836_1012 (size=1343) 2024-11-11T23:52:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741836_1012 (size=1343) 2024-11-11T23:52:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741836_1012 (size=1343) 2024-11-11T23:52:47,177 DEBUG [Thread-265 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369167116 2024-11-11T23:52:47,177 DEBUG [Thread-265 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:47,177 DEBUG [Thread-265 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:47,183 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,184 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167116) 2024-11-11T23:52:47,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=199 (was 193) Potentially hanging thread: Thread-265 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5506 (was 5513) 2024-11-11T23:52:47,200 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=199, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5505 2024-11-11T23:52:47,201 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,206 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALKeySerialization_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,207 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731369167207 2024-11-11T23:52:47,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALKeySerialization_0_ isCompressionEnabled=false]/testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731369167207 2024-11-11T23:52:47,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:47,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741837_1013 (size=93) 2024-11-11T23:52:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741837_1013 (size=93) 2024-11-11T23:52:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741837_1013 (size=93) 2024-11-11T23:52:47,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,241 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167207) 2024-11-11T23:52:47,248 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=204 (was 199) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5502 (was 5505) 2024-11-11T23:52:47,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=204, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5501 2024-11-11T23:52:47,256 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,260 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,261 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 2024-11-11T23:52:47,269 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 2024-11-11T23:52:47,270 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:47,273 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261, startPosition=0, beingWritten=true 2024-11-11T23:52:47,283 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:47,287 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,388 DEBUG [Thread-295 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261, startPosition=0, beingWritten=true 2024-11-11T23:52:47,401 DEBUG [Thread-295 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:47,401 DEBUG [Thread-295 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 to pos 460, reset compression=false 2024-11-11T23:52:47,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741838_1014 (size=468) 2024-11-11T23:52:47,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741838_1014 (size=468) 2024-11-11T23:52:47,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741838_1014 (size=468) 2024-11-11T23:52:47,413 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,413 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167261) 2024-11-11T23:52:47,421 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=212 (was 204) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Thread-295 app//org.apache.hbase.thirdparty.com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:466) app//org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.(HBaseProtos.java:27113) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.(WALProtos.java:15202) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.internalGetFieldAccessorTable(WALProtos.java:1640) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:106) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:151) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:446) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:284) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:30) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:44) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5499 (was 5501) 2024-11-11T23:52:47,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=212, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5499 2024-11-11T23:52:47,431 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,437 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437 2024-11-11T23:52:47,446 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437 2024-11-11T23:52:47,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:47,453 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,453 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437, startPosition=0, beingWritten=true 2024-11-11T23:52:47,464 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167463 2024-11-11T23:52:47,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,481 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437 with entries=3, filesize=439 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167463 2024-11-11T23:52:47,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741839_1015 (size=447) 2024-11-11T23:52:47,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741839_1015 (size=447) 2024-11-11T23:52:47,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741839_1015 (size=447) 2024-11-11T23:52:47,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:47,490 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,491 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,491 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437 for the last time to pos 321, reset compression=false 2024-11-11T23:52:47,496 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,496 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167437 2024-11-11T23:52:47,514 INFO [Thread-295 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=460, currentPosition=468 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:47,515 DEBUG [Thread-295 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 for the last time to pos 460, reset compression=false 2024-11-11T23:52:47,517 INFO [Thread-295 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 2024-11-11T23:52:47,523 INFO [Thread-295 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 2024-11-11T23:52:47,524 DEBUG [Thread-295 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 and length of the file is 468 2024-11-11T23:52:47,524 DEBUG [Thread-295 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731369167261 2024-11-11T23:52:47,525 DEBUG [Thread-295 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:47,525 DEBUG [Thread-295 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:47,596 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731369167463, startPosition=0, beingWritten=true 2024-11-11T23:52:47,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741840_1016 (size=211) 2024-11-11T23:52:47,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741840_1016 (size=211) 2024-11-11T23:52:47,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741840_1016 (size=211) 2024-11-11T23:52:47,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,615 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167463) 2024-11-11T23:52:47,624 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=217 (was 212) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5486 (was 5499) 2024-11-11T23:52:47,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=217, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5486 2024-11-11T23:52:47,632 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,636 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,637 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 2024-11-11T23:52:47,649 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 2024-11-11T23:52:47,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:47,660 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,660 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636, startPosition=0, beingWritten=true 2024-11-11T23:52:47,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167670 2024-11-11T23:52:47,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,685 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167670 2024-11-11T23:52:47,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:47,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 is not closed yet, will try archiving it next time 2024-11-11T23:52:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741841_1017 (size=218) 2024-11-11T23:52:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741841_1017 (size=218) 2024-11-11T23:52:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741841_1017 (size=218) 2024-11-11T23:52:47,691 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:47,691 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 for the last time to pos 210, reset compression=false 2024-11-11T23:52:47,695 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 and length of the file is 218 2024-11-11T23:52:47,695 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167636 2024-11-11T23:52:47,796 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731369167670, startPosition=0, beingWritten=true 2024-11-11T23:52:47,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741842_1018 (size=218) 2024-11-11T23:52:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741842_1018 (size=218) 2024-11-11T23:52:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741842_1018 (size=218) 2024-11-11T23:52:47,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:47,820 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167670) 2024-11-11T23:52:47,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=222 (was 217) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5481 (was 5486) 2024-11-11T23:52:47,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=222, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5480 2024-11-11T23:52:47,838 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:47,842 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:47,842 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731369167842 2024-11-11T23:52:47,850 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731369167842 2024-11-11T23:52:47,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:47,856 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731369167842, startPosition=0, beingWritten=true 2024-11-11T23:52:47,867 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 42 more 2024-11-11T23:52:48,868 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731369167842 to pos 203, reset compression=false 2024-11-11T23:52:48,878 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 42 more 2024-11-11T23:52:48,879 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731369167842 to pos 203, reset compression=false 2024-11-11T23:52:48,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741843_1019 (size=329) 2024-11-11T23:52:48,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741843_1019 (size=329) 2024-11-11T23:52:48,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741843_1019 (size=329) 2024-11-11T23:52:48,896 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:48,896 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369167842) 2024-11-11T23:52:48,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=227 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=5476 (was 5480) 2024-11-11T23:52:48,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=227, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5476 2024-11-11T23:52:48,913 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:48,917 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:48,917 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369168917 2024-11-11T23:52:48,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369168917 2024-11-11T23:52:48,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:48,931 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369168917, startPosition=0, beingWritten=true 2024-11-11T23:52:48,943 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369168917, startPosition=203, beingWritten=true 2024-11-11T23:52:48,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741844_1020 (size=447) 2024-11-11T23:52:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741844_1020 (size=447) 2024-11-11T23:52:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741844_1020 (size=447) 2024-11-11T23:52:48,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:48,958 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369168917) 2024-11-11T23:52:48,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=232 (was 227) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5475 (was 5476) 2024-11-11T23:52:48,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=232, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5474 2024-11-11T23:52:48,977 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:48,980 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:48,981 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 2024-11-11T23:52:48,990 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 2024-11-11T23:52:48,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:49,000 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981, startPosition=0, beingWritten=true 2024-11-11T23:52:49,009 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:49,009 DEBUG [Thread-364 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981, startPosition=0, beingWritten=true 2024-11-11T23:52:49,017 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:49,017 DEBUG [Thread-364 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 to pos 460, reset compression=false 2024-11-11T23:52:49,026 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-11T23:52:49,027 DEBUG [Thread-364 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 to pos 580, reset compression=false 2024-11-11T23:52:49,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741845_1021 (size=588) 2024-11-11T23:52:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741845_1021 (size=588) 2024-11-11T23:52:49,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741845_1021 (size=588) 2024-11-11T23:52:49,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369168981) 2024-11-11T23:52:49,035 INFO [Thread-364 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=580, currentPosition=588 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:49,037 DEBUG [Thread-364 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 for the last time to pos 580, reset compression=false 2024-11-11T23:52:49,039 INFO [Thread-364 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 2024-11-11T23:52:49,043 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=238 (was 232) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-364 app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.afterLogEvent(AwaitCompletionReliabilityStrategy.java:127) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:84) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2028) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1891) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:152) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5469 (was 5474) 2024-11-11T23:52:49,047 INFO [Thread-364 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 2024-11-11T23:52:49,048 DEBUG [Thread-364 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 and length of the file is 588 2024-11-11T23:52:49,048 DEBUG [Thread-364 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731369168981 2024-11-11T23:52:49,048 DEBUG [Thread-364 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:49,048 DEBUG [Thread-364 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:49,051 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=238, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5468 2024-11-11T23:52:49,052 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,056 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,056 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,064 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:49,066 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169066 2024-11-11T23:52:49,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,075 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169066 2024-11-11T23:52:49,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:49,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 is not closed yet, will try archiving it next time 2024-11-11T23:52:49,078 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741846_1022 (size=93) 2024-11-11T23:52:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741846_1022 (size=93) 2024-11-11T23:52:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741846_1022 (size=93) 2024-11-11T23:52:49,080 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,182 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741848_1024 (size=85) 2024-11-11T23:52:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741848_1024 (size=85) 2024-11-11T23:52:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741848_1024 (size=85) 2024-11-11T23:52:49,205 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056, startPosition=0, beingWritten=false 2024-11-11T23:52:49,207 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,212 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 for the last time to pos 0, reset compression=false 2024-11-11T23:52:49,213 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,219 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,220 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 and length of the file is 85 2024-11-11T23:52:49,220 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169056 2024-11-11T23:52:49,220 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731369169066, startPosition=0, beingWritten=true 2024-11-11T23:52:49,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741847_1023 (size=218) 2024-11-11T23:52:49,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741847_1023 (size=218) 2024-11-11T23:52:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741847_1023 (size=218) 2024-11-11T23:52:49,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,238 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369169066) 2024-11-11T23:52:49,248 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=243 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5456 (was 5468) 2024-11-11T23:52:49,256 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=243, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5456 2024-11-11T23:52:49,257 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262 2024-11-11T23:52:49,275 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262 2024-11-11T23:52:49,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:49,279 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,289 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:49,290 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,302 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:49,323 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,332 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:49,362 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,370 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:49,412 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,424 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:49,475 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,483 WARN [Thread-402 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:49,544 DEBUG [Thread-402 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262, startPosition=0, beingWritten=true 2024-11-11T23:52:49,553 DEBUG [Thread-402 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:49,553 DEBUG [Thread-402 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262 to pos 460, reset compression=false 2024-11-11T23:52:49,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741849_1025 (size=468) 2024-11-11T23:52:49,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741849_1025 (size=468) 2024-11-11T23:52:49,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741849_1025 (size=468) 2024-11-11T23:52:49,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,562 DEBUG [Thread-402 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731369169262 2024-11-11T23:52:49,562 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369169262) 2024-11-11T23:52:49,565 DEBUG [Thread-402 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:49,565 DEBUG [Thread-402 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:49,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=249 (was 243) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-402 app//org.mockito.internal.stubbing.InvocationContainerImpl.findAnswerFor(InvocationContainerImpl.java:92) app//org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:91) app//org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29) app//org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:34) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:82) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:145) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSource$MockitoMock$fUNefaxl.getSourceManager(Unknown Source) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.getSourceManager(ReplicationSourceWALReader.java:437) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.checkBufferQuota(ReplicationSourceWALReader.java:274) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:145) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5440 (was 5456) 2024-11-11T23:52:49,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=249, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5440 2024-11-11T23:52:49,582 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,586 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,587 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731369169587 2024-11-11T23:52:49,597 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731369169587 2024-11-11T23:52:49,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:49,601 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731369169587, startPosition=0, beingWritten=true 2024-11-11T23:52:49,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741850_1026 (size=93) 2024-11-11T23:52:49,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741850_1026 (size=93) 2024-11-11T23:52:49,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741850_1026 (size=93) 2024-11-11T23:52:49,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,617 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369169587) 2024-11-11T23:52:49,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=254 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5439 (was 5440) 2024-11-11T23:52:49,634 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=254, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5439 2024-11-11T23:52:49,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,639 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,640 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640 2024-11-11T23:52:49,651 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640 2024-11-11T23:52:49,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:49,657 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640, startPosition=0, beingWritten=true 2024-11-11T23:52:49,669 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:49,670 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640, startPosition=210, beingWritten=true 2024-11-11T23:52:49,679 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169678 2024-11-11T23:52:49,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,695 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640 with entries=3, filesize=460 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169678 2024-11-11T23:52:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741851_1027 (size=468) 2024-11-11T23:52:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741851_1027 (size=468) 2024-11-11T23:52:49,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741851_1027 (size=468) 2024-11-11T23:52:49,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:49,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:49,706 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640, startPosition=335, beingWritten=false 2024-11-11T23:52:49,710 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:49,710 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169640 2024-11-11T23:52:49,811 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731369169678, startPosition=0, beingWritten=true 2024-11-11T23:52:49,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741852_1028 (size=218) 2024-11-11T23:52:49,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741852_1028 (size=218) 2024-11-11T23:52:49,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741852_1028 (size=218) 2024-11-11T23:52:49,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,831 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369169678) 2024-11-11T23:52:49,841 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=259 (was 254) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5422 (was 5439) 2024-11-11T23:52:49,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=259, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5422 2024-11-11T23:52:49,851 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,855 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,856 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369169856 2024-11-11T23:52:49,892 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369169856 2024-11-11T23:52:49,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:49,918 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369169856, startPosition=0, beingWritten=true 2024-11-11T23:52:49,930 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731369169856, startPosition=210, beingWritten=true 2024-11-11T23:52:49,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:49,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741853_1029 (size=468) 2024-11-11T23:52:49,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741853_1029 (size=468) 2024-11-11T23:52:49,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741853_1029 (size=468) 2024-11-11T23:52:49,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:49,951 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369169856) 2024-11-11T23:52:49,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=263 (was 259) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=5397 (was 5422) 2024-11-11T23:52:49,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=263, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=12, AvailableMemoryMB=5395 2024-11-11T23:52:49,974 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:49,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:49,979 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978 2024-11-11T23:52:49,993 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978 2024-11-11T23:52:49,998 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:50,001 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369170001 2024-11-11T23:52:50,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,013 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369170001 2024-11-11T23:52:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741854_1030 (size=218) 2024-11-11T23:52:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741854_1030 (size=218) 2024-11-11T23:52:50,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741854_1030 (size=218) 2024-11-11T23:52:50,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:50,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978 is not closed yet, will try archiving it next time 2024-11-11T23:52:50,018 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:50,021 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978, startPosition=0, beingWritten=false 2024-11-11T23:52:50,025 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731369169978 2024-11-11T23:52:50,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741855_1031 (size=93) 2024-11-11T23:52:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741855_1031 (size=93) 2024-11-11T23:52:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741855_1031 (size=93) 2024-11-11T23:52:50,036 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,036 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731369170001) 2024-11-11T23:52:50,049 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=269 (was 263) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 12), AvailableMemoryMB=5389 (was 5395) 2024-11-11T23:52:50,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=269, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=12, AvailableMemoryMB=5387 2024-11-11T23:52:50,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,066 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,067 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369170067 2024-11-11T23:52:50,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369170067 2024-11-11T23:52:50,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,096 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369170067, startPosition=0, beingWritten=true 2024-11-11T23:52:50,102 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369170067: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,108 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369170067 to pos 197, reset compression=false 2024-11-11T23:52:50,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741856_1032 (size=317) 2024-11-11T23:52:50,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741856_1032 (size=317) 2024-11-11T23:52:50,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741856_1032 (size=317) 2024-11-11T23:52:50,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,123 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170067) 2024-11-11T23:52:50,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=274 (was 269) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=12 (was 12), AvailableMemoryMB=5381 (was 5387) 2024-11-11T23:52:50,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=274, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=12, AvailableMemoryMB=5379 2024-11-11T23:52:50,144 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,149 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 2024-11-11T23:52:50,167 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 2024-11-11T23:52:50,172 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,179 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150, startPosition=0, beingWritten=true 2024-11-11T23:52:50,191 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,195 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:50,198 DEBUG [Thread-484 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150, startPosition=0, beingWritten=true 2024-11-11T23:52:50,204 DEBUG [Thread-484 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,207 WARN [Thread-484 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:50,228 DEBUG [Thread-484 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150, startPosition=0, beingWritten=true 2024-11-11T23:52:50,234 DEBUG [Thread-484 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,237 WARN [Thread-484 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:50,268 DEBUG [Thread-484 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150, startPosition=0, beingWritten=true 2024-11-11T23:52:50,273 DEBUG [Thread-484 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,277 WARN [Thread-484 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-11T23:52:50,318 DEBUG [Thread-484 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150, startPosition=0, beingWritten=true 2024-11-11T23:52:50,326 DEBUG [Thread-484 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,329 DEBUG [Thread-484 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:50,329 DEBUG [Thread-484 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 to pos 314, reset compression=false 2024-11-11T23:52:50,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741857_1033 (size=322) 2024-11-11T23:52:50,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741857_1033 (size=322) 2024-11-11T23:52:50,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741857_1033 (size=322) 2024-11-11T23:52:50,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,346 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170150) 2024-11-11T23:52:50,355 DEBUG [Thread-484 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 to pos 314, reset compression=false 2024-11-11T23:52:50,356 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=280 (was 274) Potentially hanging thread: Thread-484 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 12), AvailableMemoryMB=5360 (was 5379) 2024-11-11T23:52:50,361 INFO [Thread-484 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 2024-11-11T23:52:50,364 DEBUG [Thread-484 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369170150 2024-11-11T23:52:50,364 DEBUG [Thread-484 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:50,364 DEBUG [Thread-484 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:50,365 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=280, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5360 2024-11-11T23:52:50,366 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,369 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,370 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731369170370 2024-11-11T23:52:50,384 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:52:50,387 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731369170370 2024-11-11T23:52:50,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,398 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.true 2024-11-11T23:52:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741859_1035 (size=225) 2024-11-11T23:52:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741859_1035 (size=225) 2024-11-11T23:52:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741859_1035 (size=225) 2024-11-11T23:52:50,417 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-11T23:52:50,418 DEBUG [Thread-508 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/user/jenkins/log.1.true, startPosition=0, beingWritten=false 2024-11-11T23:52:50,421 DEBUG [Thread-508 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/user/jenkins/log.1.true: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,422 DEBUG [Thread-508 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/user/jenkins/log.1.true 2024-11-11T23:52:50,423 DEBUG [Thread-508 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:50,423 DEBUG [Thread-508 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/user/jenkins/log.2.true, startPosition=0, beingWritten=false 2024-11-11T23:52:50,424 WARN [Thread-508 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38259/user/jenkins/log.2.true, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:50,424 DEBUG [Thread-508 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/user/jenkins/log.2.true 2024-11-11T23:52:50,424 DEBUG [Thread-508 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:50,424 DEBUG [Thread-508 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:50,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741858_1034 (size=93) 2024-11-11T23:52:50,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741858_1034 (size=93) 2024-11-11T23:52:50,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741858_1034 (size=93) 2024-11-11T23:52:50,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,431 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170370) 2024-11-11T23:52:50,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=286 (was 280) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5348 (was 5360) 2024-11-11T23:52:50,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=286, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5347 2024-11-11T23:52:50,446 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,449 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,450 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,459 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 2024-11-11T23:52:50,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,477 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 2024-11-11T23:52:50,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 is not closed yet, will try archiving it next time 2024-11-11T23:52:50,478 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:50,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741860_1036 (size=93) 2024-11-11T23:52:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741860_1036 (size=93) 2024-11-11T23:52:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741860_1036 (size=93) 2024-11-11T23:52:50,482 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,582 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,593 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:50,593 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:50,594 DEBUG [Thread-525 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450, startPosition=0, beingWritten=false 2024-11-11T23:52:50,595 INFO [Thread-525 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,596 WARN [Thread-525 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:50,597 DEBUG [Thread-525 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170450 2024-11-11T23:52:50,597 DEBUG [Thread-525 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469, startPosition=0, beingWritten=true 2024-11-11T23:52:50,602 DEBUG [Thread-525 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,613 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,632 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,648 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,664 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,685 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741861_1037 (size=93) 2024-11-11T23:52:50,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741861_1037 (size=93) 2024-11-11T23:52:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741861_1037 (size=93) 2024-11-11T23:52:50,702 DEBUG [Thread-525 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 to pos 0, reset compression=false 2024-11-11T23:52:50,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,703 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170469) 2024-11-11T23:52:50,707 INFO [Thread-525 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 2024-11-11T23:52:50,711 DEBUG [Thread-525 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731369170469 2024-11-11T23:52:50,711 DEBUG [Thread-525 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:50,711 DEBUG [Thread-525 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:50,713 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=292 (was 286) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-525 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$341/0x00007f799487ab98.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5339 (was 5347) 2024-11-11T23:52:50,723 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=292, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5339 2024-11-11T23:52:50,724 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,728 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729 2024-11-11T23:52:50,738 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729 2024-11-11T23:52:50,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:50,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170744 2024-11-11T23:52:50,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,755 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729 with entries=10, filesize=707 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170744 2024-11-11T23:52:50,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:50,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729 is not closed yet, will try archiving it next time 2024-11-11T23:52:50,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741862_1038 (size=715) 2024-11-11T23:52:50,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741862_1038 (size=715) 2024-11-11T23:52:50,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741862_1038 (size=715) 2024-11-11T23:52:50,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741863_1039 (size=440) 2024-11-11T23:52:50,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741863_1039 (size=440) 2024-11-11T23:52:50,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741863_1039 (size=440) 2024-11-11T23:52:50,764 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-11T23:52:50,765 DEBUG [Thread-548 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729, startPosition=0, beingWritten=false 2024-11-11T23:52:50,767 DEBUG [Thread-548 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,770 DEBUG [Thread-548 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-11T23:52:50,771 DEBUG [Thread-548 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170729 2024-11-11T23:52:50,771 DEBUG [Thread-548 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170744, startPosition=0, beingWritten=false 2024-11-11T23:52:50,774 DEBUG [Thread-548 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170744: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,777 DEBUG [Thread-548 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731369170744 2024-11-11T23:52:50,777 DEBUG [Thread-548 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-11T23:52:50,777 DEBUG [Thread-548 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:50,777 DEBUG [Thread-548 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:50,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,781 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170744) 2024-11-11T23:52:50,789 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=297 (was 292) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5331 (was 5339) 2024-11-11T23:52:50,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=297, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=12, AvailableMemoryMB=5329 2024-11-11T23:52:50,799 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,804 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369170804 2024-11-11T23:52:50,818 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369170804 2024-11-11T23:52:50,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,827 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-11T23:52:50,828 DEBUG [Thread-558 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-11T23:52:50,829 WARN [Thread-558 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-11T23:52:50,830 DEBUG [Thread-558 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-11T23:52:50,830 DEBUG [Thread-558 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:50,830 DEBUG [Thread-558 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:50,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741864_1040 (size=93) 2024-11-11T23:52:50,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741864_1040 (size=93) 2024-11-11T23:52:50,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741864_1040 (size=93) 2024-11-11T23:52:50,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:50,839 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170804) 2024-11-11T23:52:50,849 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=302 (was 297) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 12), AvailableMemoryMB=5323 (was 5329) 2024-11-11T23:52:50,858 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=302, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5323 2024-11-11T23:52:50,859 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:50,861 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:50,862 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862 2024-11-11T23:52:50,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862 2024-11-11T23:52:50,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,872 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 2024-11-11T23:52:50,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,880 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 2024-11-11T23:52:50,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:50,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862 is not closed yet, will try archiving it next time 2024-11-11T23:52:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741865_1041 (size=212) 2024-11-11T23:52:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741865_1041 (size=212) 2024-11-11T23:52:50,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741865_1041 (size=212) 2024-11-11T23:52:50,886 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:50,887 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:50,888 DEBUG [Thread-575 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862, startPosition=0, beingWritten=false 2024-11-11T23:52:50,890 DEBUG [Thread-575 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,892 DEBUG [Thread-575 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170862 2024-11-11T23:52:50,892 DEBUG [Thread-575 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-11T23:52:50,892 DEBUG [Thread-575 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872, startPosition=0, beingWritten=true 2024-11-11T23:52:50,896 DEBUG [Thread-575 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,903 DEBUG [Thread-575 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-11T23:52:50,903 DEBUG [Thread-575 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 to pos 1281, reset compression=false 2024-11-11T23:52:50,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 2024-11-11T23:52:50,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,913 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 with entries=20, filesize=1.25 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 2024-11-11T23:52:50,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:50,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 is not closed yet, will try archiving it next time 2024-11-11T23:52:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741866_1042 (size=1289) 2024-11-11T23:52:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741866_1042 (size=1289) 2024-11-11T23:52:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741866_1042 (size=1289) 2024-11-11T23:52:50,934 DEBUG [Thread-575 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 to pos 1281, reset compression=false 2024-11-11T23:52:50,937 DEBUG [Thread-575 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170872 2024-11-11T23:52:50,937 DEBUG [Thread-575 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903, startPosition=0, beingWritten=true 2024-11-11T23:52:50,944 DEBUG [Thread-575 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:50,950 DEBUG [Thread-575 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-11T23:52:50,950 DEBUG [Thread-575 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 to pos 715, reset compression=false 2024-11-11T23:52:50,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:50,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741867_1043 (size=723) 2024-11-11T23:52:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741867_1043 (size=723) 2024-11-11T23:52:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741867_1043 (size=723) 2024-11-11T23:52:51,321 INFO [Thread-575 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=715, currentPosition=723 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:51,322 DEBUG [Thread-575 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 for the last time to pos 715, reset compression=false 2024-11-11T23:52:51,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:51,325 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369170903) 2024-11-11T23:52:51,329 INFO [Thread-575 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 2024-11-11T23:52:51,330 DEBUG [Thread-575 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 and length of the file is 723 2024-11-11T23:52:51,330 DEBUG [Thread-575 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369170903 2024-11-11T23:52:51,330 DEBUG [Thread-575 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:51,330 DEBUG [Thread-575 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:51,334 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=308 (was 302) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #64] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-575 java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:163) app//org.apache.hadoop.ipc.Client.isAsynchronousMode(Client.java:1541) app//org.apache.hadoop.ipc.Client.checkAsyncCall(Client.java:1431) app//org.apache.hadoop.ipc.Client.call(Client.java:1477) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$311/0x00007f79947b7418.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) jdk.internal.reflect.GeneratedMethodAccessor30.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.getCurrentPathFileStatus(WALEntryStream.java:399) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.checkAllBytesParsed(WALEntryStream.java:417) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:315) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5304 (was 5323) 2024-11-11T23:52:51,347 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=308, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5303 2024-11-11T23:52:51,348 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:51,352 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALKeySerialization_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:51,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731369171353 2024-11-11T23:52:51,363 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALKeySerialization_1_ isCompressionEnabled=true]/testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731369171353 2024-11-11T23:52:51,364 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:51,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741868_1044 (size=93) 2024-11-11T23:52:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741868_1044 (size=93) 2024-11-11T23:52:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741868_1044 (size=93) 2024-11-11T23:52:51,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:51,371 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369171353) 2024-11-11T23:52:51,381 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=313 (was 308) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5299 (was 5303) 2024-11-11T23:52:51,389 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=313, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5298 2024-11-11T23:52:51,389 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:51,392 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:51,393 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 2024-11-11T23:52:51,399 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 2024-11-11T23:52:51,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:51,402 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392, startPosition=0, beingWritten=true 2024-11-11T23:52:51,407 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,410 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:51,410 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,513 DEBUG [Thread-605 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392, startPosition=0, beingWritten=true 2024-11-11T23:52:51,524 DEBUG [Thread-605 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,527 DEBUG [Thread-605 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:51,527 DEBUG [Thread-605 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 to pos 322, reset compression=false 2024-11-11T23:52:51,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741869_1045 (size=330) 2024-11-11T23:52:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741869_1045 (size=330) 2024-11-11T23:52:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741869_1045 (size=330) 2024-11-11T23:52:51,536 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:51,536 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369171392) 2024-11-11T23:52:51,536 INFO [Thread-605 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=322, currentPosition=330 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:51,539 DEBUG [Thread-605 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 for the last time to pos 322, reset compression=false 2024-11-11T23:52:51,540 INFO [Thread-605 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 2024-11-11T23:52:51,544 INFO [Thread-605 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 2024-11-11T23:52:51,544 DEBUG [Thread-605 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 and length of the file is 330 2024-11-11T23:52:51,544 DEBUG [Thread-605 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731369171392 2024-11-11T23:52:51,544 DEBUG [Thread-605 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:51,544 DEBUG [Thread-605 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:51,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=320 (was 313) Potentially hanging thread: Thread-605 java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeBufferLoop(UTF_8.java:555) java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:566) java.base@17.0.11/java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:585) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeString(AbstractStreamEncoder.java:118) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeStringData(AbstractStreamEncoder.java:156) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encode(AbstractStreamEncoder.java:145) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.encodeMessage(EventChannelEncoder.java:378) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:172) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2028) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1891) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:152) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #58] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #67] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5291 (was 5298) 2024-11-11T23:52:51,554 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=320, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5290 2024-11-11T23:52:51,555 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:51,558 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:51,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 2024-11-11T23:52:51,566 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 2024-11-11T23:52:51,567 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:51,571 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,571 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558, startPosition=0, beingWritten=true 2024-11-11T23:52:51,576 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171580 2024-11-11T23:52:51,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,590 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 with entries=3, filesize=309 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171580 2024-11-11T23:52:51,592 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:51,592 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 is not closed yet, will try archiving it next time 2024-11-11T23:52:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741870_1046 (size=317) 2024-11-11T23:52:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741870_1046 (size=317) 2024-11-11T23:52:51,594 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741870_1046 (size=317) 2024-11-11T23:52:51,595 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,595 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 for the last time to pos 253, reset compression=false 2024-11-11T23:52:51,600 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,601 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171558 2024-11-11T23:52:51,701 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171580, startPosition=0, beingWritten=true 2024-11-11T23:52:51,706 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731369171580: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741871_1047 (size=205) 2024-11-11T23:52:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741871_1047 (size=205) 2024-11-11T23:52:51,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741871_1047 (size=205) 2024-11-11T23:52:51,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:51,715 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369171580) 2024-11-11T23:52:51,723 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=325 (was 320) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #62] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #69] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5282 (was 5290) 2024-11-11T23:52:51,733 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=325, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5282 2024-11-11T23:52:51,734 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:51,737 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:51,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 2024-11-11T23:52:51,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 2024-11-11T23:52:51,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:51,750 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,750 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737, startPosition=0, beingWritten=true 2024-11-11T23:52:51,754 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,756 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171756 2024-11-11T23:52:51,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,764 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171756 2024-11-11T23:52:51,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:51,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 is not closed yet, will try archiving it next time 2024-11-11T23:52:51,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741872_1048 (size=212) 2024-11-11T23:52:51,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741872_1048 (size=212) 2024-11-11T23:52:51,767 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:51,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741872_1048 (size=212) 2024-11-11T23:52:51,767 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 for the last time to pos 204, reset compression=false 2024-11-11T23:52:51,770 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 and length of the file is 212 2024-11-11T23:52:51,770 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171737 2024-11-11T23:52:51,871 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171756, startPosition=0, beingWritten=true 2024-11-11T23:52:51,879 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731369171756: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:51,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741873_1049 (size=212) 2024-11-11T23:52:51,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741873_1049 (size=212) 2024-11-11T23:52:51,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741873_1049 (size=212) 2024-11-11T23:52:51,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:51,892 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369171756) 2024-11-11T23:52:51,903 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=330 (was 325) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #64] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #72] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5274 (was 5282) 2024-11-11T23:52:51,913 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=330, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=5274 2024-11-11T23:52:51,913 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:51,917 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:51,918 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918 2024-11-11T23:52:51,928 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918 2024-11-11T23:52:51,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:51,940 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918, startPosition=0, beingWritten=true 2024-11-11T23:52:51,947 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:51,948 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 42 more 2024-11-11T23:52:52,949 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918 to pos 197, reset compression=true 2024-11-11T23:52:52,957 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 42 more 2024-11-11T23:52:52,958 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731369171918 to pos 197, reset compression=true 2024-11-11T23:52:52,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:52,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:52,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:52,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:52,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:52,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741874_1050 (size=261) 2024-11-11T23:52:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741874_1050 (size=261) 2024-11-11T23:52:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741874_1050 (size=261) 2024-11-11T23:52:52,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:52,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369171918) 2024-11-11T23:52:52,981 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=337 (was 330) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #66] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #74] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 199), ProcessCount=11 (was 11), AvailableMemoryMB=5258 (was 5274) 2024-11-11T23:52:52,990 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=337, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5257 2024-11-11T23:52:52,991 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:52,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:52,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994 2024-11-11T23:52:53,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994 2024-11-11T23:52:53,005 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:53,010 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994, startPosition=0, beingWritten=true 2024-11-11T23:52:53,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,022 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994, startPosition=197, beingWritten=true 2024-11-11T23:52:53,031 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369172994: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741875_1051 (size=317) 2024-11-11T23:52:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741875_1051 (size=317) 2024-11-11T23:52:53,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741875_1051 (size=317) 2024-11-11T23:52:53,039 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:53,039 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369172994) 2024-11-11T23:52:53,051 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=342 (was 337) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #78] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5252 (was 5257) 2024-11-11T23:52:53,061 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=342, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5251 2024-11-11T23:52:53,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:53,065 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:53,066 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 2024-11-11T23:52:53,073 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 2024-11-11T23:52:53,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:53,082 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066, startPosition=0, beingWritten=true 2024-11-11T23:52:53,087 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,090 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:53,092 DEBUG [Thread-676 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066, startPosition=0, beingWritten=true 2024-11-11T23:52:53,096 DEBUG [Thread-676 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,098 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:53,098 DEBUG [Thread-676 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 to pos 322, reset compression=false 2024-11-11T23:52:53,108 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-11T23:52:53,108 DEBUG [Thread-676 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 to pos 380, reset compression=false 2024-11-11T23:52:53,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741876_1052 (size=388) 2024-11-11T23:52:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741876_1052 (size=388) 2024-11-11T23:52:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741876_1052 (size=388) 2024-11-11T23:52:53,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:53,116 INFO [Thread-676 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=380, currentPosition=388 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:53,116 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369173066) 2024-11-11T23:52:53,116 DEBUG [Thread-676 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 for the last time to pos 380, reset compression=false 2024-11-11T23:52:53,121 INFO [Thread-676 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 2024-11-11T23:52:53,124 INFO [Thread-676 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 2024-11-11T23:52:53,124 DEBUG [Thread-676 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 and length of the file is 388 2024-11-11T23:52:53,124 DEBUG [Thread-676 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731369173066 2024-11-11T23:52:53,125 DEBUG [Thread-676 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:53,125 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:53,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=348 (was 342) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #73] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741876_1052] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-676 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$341/0x00007f799487ab98.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5235 (was 5251) 2024-11-11T23:52:53,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=348, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5235 2024-11-11T23:52:53,140 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:53,143 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:53,145 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,155 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:53,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173165 2024-11-11T23:52:53,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,183 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173165 2024-11-11T23:52:53,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741877_1053 (size=93) 2024-11-11T23:52:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741877_1053 (size=93) 2024-11-11T23:52:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741877_1053 (size=93) 2024-11-11T23:52:53,189 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:53,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:53,196 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741879_1055 (size=85) 2024-11-11T23:52:53,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741879_1055 (size=85) 2024-11-11T23:52:53,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741879_1055 (size=85) 2024-11-11T23:52:53,220 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144, startPosition=0, beingWritten=false 2024-11-11T23:52:53,222 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,225 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,226 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 for the last time to pos 0, reset compression=false 2024-11-11T23:52:53,230 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,239 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,239 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 and length of the file is 85 2024-11-11T23:52:53,239 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173144 2024-11-11T23:52:53,240 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173165, startPosition=0, beingWritten=true 2024-11-11T23:52:53,245 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731369173165: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741878_1054 (size=212) 2024-11-11T23:52:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741878_1054 (size=212) 2024-11-11T23:52:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741878_1054 (size=212) 2024-11-11T23:52:53,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:53,656 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369173165) 2024-11-11T23:52:53,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=353 (was 348) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #75] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Waiting for operation #83] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5206 (was 5235) 2024-11-11T23:52:53,680 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=353, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5206 2024-11-11T23:52:53,681 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:53,685 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:53,686 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 2024-11-11T23:52:53,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 2024-11-11T23:52:53,718 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:53,724 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686, startPosition=0, beingWritten=true 2024-11-11T23:52:53,734 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,738 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T23:52:53,738 DEBUG [Thread-714 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686, startPosition=0, beingWritten=true 2024-11-11T23:52:53,745 DEBUG [Thread-714 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,747 DEBUG [Thread-714 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-11T23:52:53,747 DEBUG [Thread-714 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 to pos 322, reset compression=false 2024-11-11T23:52:53,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741880_1056 (size=330) 2024-11-11T23:52:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741880_1056 (size=330) 2024-11-11T23:52:53,753 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785):Got exception while serving BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056 to /127.0.0.1:42022 java.io.FileNotFoundException: BlockId 1073741880 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:52:53,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741880_1056 (size=330) 2024-11-11T23:52:53,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42022 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056] {}] datanode.DataXceiver(331): 127.0.0.1:38527:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:42022 dst: /127.0.0.1:38527 java.io.FileNotFoundException: BlockId 1073741880 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:52:53,758 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:46730 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:38527, datanodeUuid=27c33c56-7aed-459b-b28a-e36c927d58af, infoPort=37751, infoSecurePort=0, ipcPort=37305, storageInfo=lv=-57;cid=testClusterID;nsid=36597849;c=1731369161785):Got exception while serving BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056 to /127.0.0.1:46730 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:52:53,758 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:46730 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056] {}] datanode.DataXceiver(331): 127.0.0.1:38527:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:46730 dst: /127.0.0.1:38527 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:52:53,758 WARN [Thread-714 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:46730, remote=/127.0.0.1:38527, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686, for pool BP-740878948-172.17.0.2-1731369161785 block 1073741880_1056 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:53,759 WARN [Thread-714 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:38527 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 for block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:46730, remote=/127.0.0.1:38527, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686, for pool BP-740878948-172.17.0.2-1731369161785 block 1073741880_1056 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-11T23:52:53,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:53,760 DEBUG [Thread-714 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 for the last time to pos 322, reset compression=false 2024-11-11T23:52:53,760 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369173686) 2024-11-11T23:52:53,764 INFO [Thread-714 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 2024-11-11T23:52:53,768 INFO [Thread-714 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 was moved to hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 2024-11-11T23:52:53,768 DEBUG [Thread-714 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 and length of the file is 330 2024-11-11T23:52:53,768 DEBUG [Thread-714 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731369173686 2024-11-11T23:52:53,769 DEBUG [Thread-714 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-11T23:52:53,769 DEBUG [Thread-714 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-11T23:52:53,772 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=358 (was 353) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Sending block BP-740878948-172.17.0.2-1731369161785:blk_1073741880_1056] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-714 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy48.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$341/0x00007f799487ab98.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy49.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:42076 [Waiting for operation #60] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5162 (was 5206) 2024-11-11T23:52:53,782 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=358, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5162 2024-11-11T23:52:53,782 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:53,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:53,786 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731369173786 2024-11-11T23:52:53,794 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731369173786 2024-11-11T23:52:53,798 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:37751:37751)] 2024-11-11T23:52:53,799 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731369173786, startPosition=0, beingWritten=true 2024-11-11T23:52:53,812 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731369173786: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741881_1057 (size=93) 2024-11-11T23:52:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741881_1057 (size=93) 2024-11-11T23:52:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741881_1057 (size=93) 2024-11-11T23:52:53,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:53,826 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369173786) 2024-11-11T23:52:53,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=364 (was 358) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:46744 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5158 (was 5162) 2024-11-11T23:52:53,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=364, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5157 2024-11-11T23:52:53,847 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:53,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:53,851 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850 2024-11-11T23:52:53,861 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850 2024-11-11T23:52:53,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:53,870 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850, startPosition=0, beingWritten=true 2024-11-11T23:52:53,876 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,879 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:53,879 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850, startPosition=204, beingWritten=true 2024-11-11T23:52:53,883 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173885 2024-11-11T23:52:53,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:53,895 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850 with entries=3, filesize=330 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173885 2024-11-11T23:52:53,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:53,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850 is not closed yet, will try archiving it next time 2024-11-11T23:52:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741882_1058 (size=338) 2024-11-11T23:52:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741882_1058 (size=338) 2024-11-11T23:52:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741882_1058 (size=338) 2024-11-11T23:52:53,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:53,900 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850, startPosition=267, beingWritten=false 2024-11-11T23:52:53,902 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:53,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:53,904 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173850 2024-11-11T23:52:54,004 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173885, startPosition=0, beingWritten=true 2024-11-11T23:52:54,009 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731369173885: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:54,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741883_1059 (size=212) 2024-11-11T23:52:54,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741883_1059 (size=212) 2024-11-11T23:52:54,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741883_1059 (size=212) 2024-11-11T23:52:54,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:54,016 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369173885) 2024-11-11T23:52:54,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=369 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #80] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:46744 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5155 (was 5157) 2024-11-11T23:52:54,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=369, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5155 2024-11-11T23:52:54,038 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:54,041 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:54,041 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041 2024-11-11T23:52:54,049 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041 2024-11-11T23:52:54,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:54,056 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041, startPosition=0, beingWritten=true 2024-11-11T23:52:54,061 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:54,062 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041, startPosition=204, beingWritten=true 2024-11-11T23:52:54,067 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731369174041: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:54,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741884_1060 (size=322) 2024-11-11T23:52:54,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741884_1060 (size=322) 2024-11-11T23:52:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741884_1060 (size=322) 2024-11-11T23:52:54,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:54,074 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369174041) 2024-11-11T23:52:54,083 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=374 (was 369) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #82] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:46744 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5151 (was 5155) 2024-11-11T23:52:54,094 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=374, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=5149 2024-11-11T23:52:54,095 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-11T23:52:54,098 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs, maxLogs=32 2024-11-11T23:52:54,098 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098 2024-11-11T23:52:54,105 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098 2024-11-11T23:52:54,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38179:38179),(127.0.0.1/127.0.0.1:38515:38515)] 2024-11-11T23:52:54,112 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174112 2024-11-11T23:52:54,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,121 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174112 2024-11-11T23:52:54,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751),(127.0.0.1/127.0.0.1:38515:38515),(127.0.0.1/127.0.0.1:38179:38179)] 2024-11-11T23:52:54,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098 is not closed yet, will try archiving it next time 2024-11-11T23:52:54,123 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:52:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741885_1061 (size=212) 2024-11-11T23:52:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741885_1061 (size=212) 2024-11-11T23:52:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741885_1061 (size=212) 2024-11-11T23:52:54,224 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098, startPosition=0, beingWritten=false 2024-11-11T23:52:54,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-11T23:52:54,231 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:38259/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731369174098 2024-11-11T23:52:54,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T23:52:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37561 is added to blk_1073741886_1062 (size=93) 2024-11-11T23:52:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741886_1062 (size=93) 2024-11-11T23:52:54,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38527 is added to blk_1073741886_1062 (size=93) 2024-11-11T23:52:54,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/oldWALs 2024-11-11T23:52:54,241 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731369174112) 2024-11-11T23:52:54,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=379 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_706526272_22 at /127.0.0.1:48342 [Waiting for operation #84] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=5149 (was 5149) 2024-11-11T23:52:54,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T23:52:54,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d954ade{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:54,260 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:52:54,261 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:52:54,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:52:54,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,STOPPED} 2024-11-11T23:52:54,263 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T23:52:54,263 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T23:52:54,263 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-740878948-172.17.0.2-1731369161785 (Datanode Uuid 1e8c5503-00ac-424f-a1d0-54cf5db073a2) service to localhost/127.0.0.1:38259 2024-11-11T23:52:54,263 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T23:52:54,265 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data5/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,265 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data6/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,265 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T23:52:54,270 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fecc8a3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:54,271 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:52:54,271 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:52:54,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:52:54,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,STOPPED} 2024-11-11T23:52:54,272 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T23:52:54,272 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T23:52:54,272 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-740878948-172.17.0.2-1731369161785 (Datanode Uuid 27c33c56-7aed-459b-b28a-e36c927d58af) service to localhost/127.0.0.1:38259 2024-11-11T23:52:54,272 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T23:52:54,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data3/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data4/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,274 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T23:52:54,276 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a86aa33{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:52:54,277 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:52:54,277 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:52:54,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:52:54,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,STOPPED} 2024-11-11T23:52:54,279 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T23:52:54,279 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T23:52:54,279 WARN [BP-740878948-172.17.0.2-1731369161785 heartbeating to localhost/127.0.0.1:38259 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-740878948-172.17.0.2-1731369161785 (Datanode Uuid 70be881c-ee35-4e55-8d9a-bf699cd3174a) service to localhost/127.0.0.1:38259 2024-11-11T23:52:54,279 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T23:52:54,279 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data1/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,280 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/cluster_4f83d4ec-753c-b344-6269-2f288475d15d/data/data2/current/BP-740878948-172.17.0.2-1731369161785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:52:54,280 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T23:52:54,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44567ea4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:52:54,288 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:52:54,288 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:52:54,289 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:52:54,289 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/240cc706-ee38-6884-62b2-6611a83670f6/hadoop.log.dir/,STOPPED} 2024-11-11T23:52:54,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down