2024-04-12 03:12:11,831 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 2024-04-12 03:12:11,850 main DEBUG Took 0.016016 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-04-12 03:12:11,851 main DEBUG PluginManager 'Core' found 129 plugins 2024-04-12 03:12:11,851 main DEBUG PluginManager 'Level' found 0 plugins 2024-04-12 03:12:11,852 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-04-12 03:12:11,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,873 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-04-12 03:12:11,891 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,894 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,895 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,897 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,898 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,899 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,900 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,901 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,902 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,902 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:11,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,905 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-04-12 03:12:11,908 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:11,911 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-04-12 03:12:11,913 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-04-12 03:12:11,914 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-04-12 03:12:11,915 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-04-12 03:12:11,916 main DEBUG PluginManager 'Converter' found 47 plugins 2024-04-12 03:12:11,931 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-04-12 03:12:11,935 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-04-12 03:12:11,940 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-04-12 03:12:11,941 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-04-12 03:12:11,941 main DEBUG createAppenders(={Console}) 2024-04-12 03:12:11,942 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 initialized 2024-04-12 03:12:11,943 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 2024-04-12 03:12:11,943 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 OK. 2024-04-12 03:12:11,944 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-04-12 03:12:11,944 main DEBUG OutputStream closed 2024-04-12 03:12:11,945 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-04-12 03:12:11,945 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-04-12 03:12:11,945 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55141def OK 2024-04-12 03:12:12,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-04-12 03:12:12,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-04-12 03:12:12,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-04-12 03:12:12,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-04-12 03:12:12,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-04-12 03:12:12,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-04-12 03:12:12,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-04-12 03:12:12,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-04-12 03:12:12,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-04-12 03:12:12,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-04-12 03:12:12,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-04-12 03:12:12,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-04-12 03:12:12,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-04-12 03:12:12,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-04-12 03:12:12,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-04-12 03:12:12,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-04-12 03:12:12,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-04-12 03:12:12,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-04-12 03:12:12,022 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-12 03:12:12,022 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@73e9cf30) with optional ClassLoader: null 2024-04-12 03:12:12,023 main DEBUG Shutdown hook enabled. Registering a new one. 2024-04-12 03:12:12,024 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@73e9cf30] started OK. 2024-04-12T03:12:12,354 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293 2024-04-12 03:12:12,359 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-04-12 03:12:12,359 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-12T03:12:12,404 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL timeout: 104 mins 2024-04-12T03:12:12,416 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33, deleteOnExit=true 2024-04-12T03:12:12,417 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/test.cache.data in system properties and HBase conf 2024-04-12T03:12:12,418 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/hadoop.tmp.dir in system properties and HBase conf 2024-04-12T03:12:12,419 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/hadoop.log.dir in system properties and HBase conf 2024-04-12T03:12:12,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/mapreduce.cluster.local.dir in system properties and HBase conf 2024-04-12T03:12:12,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-04-12T03:12:12,421 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-04-12T03:12:12,556 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-04-12T03:12:13,128 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:13,134 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-04-12T03:12:13,134 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-04-12T03:12:13,135 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-04-12T03:12:13,135 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-12T03:12:13,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-04-12T03:12:13,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-04-12T03:12:13,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-12T03:12:13,137 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-12T03:12:13,137 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-04-12T03:12:13,138 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/nfs.dump.dir in system properties and HBase conf 2024-04-12T03:12:13,138 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/java.io.tmpdir in system properties and HBase conf 2024-04-12T03:12:13,138 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-12T03:12:13,139 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-04-12T03:12:13,139 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2024-04-12T03:12:13,675 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-12T03:12:13,680 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-12T03:12:13,960 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-04-12T03:12:14,086 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@32a7bad7 via org.mortbay.log.Slf4jLog 2024-04-12T03:12:14,100 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-12T03:12:14,137 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-12T03:12:14,178 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/java.io.tmpdir/Jetty_localhost_40737_hdfs____.rb9nxz/webapp 2024-04-12T03:12:14,316 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40737 2024-04-12T03:12:14,338 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-12T03:12:14,338 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-12T03:12:14,946 WARN [Listener at localhost/33707 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-12T03:12:15,019 WARN [Listener at localhost/33707 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-12T03:12:15,039 WARN [Listener at localhost/33707 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-12T03:12:15,046 INFO [Listener at localhost/33707 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-12T03:12:15,054 INFO [Listener at localhost/33707 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/java.io.tmpdir/Jetty_localhost_35775_datanode____.ur3pba/webapp 2024-04-12T03:12:15,172 INFO [Listener at localhost/33707 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35775 2024-04-12T03:12:15,415 WARN [Listener at localhost/42429 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-12T03:12:15,432 WARN [Listener at localhost/42429 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-12T03:12:15,437 WARN [Listener at localhost/42429 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-12T03:12:15,439 INFO [Listener at localhost/42429 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-12T03:12:15,449 INFO [Listener at localhost/42429 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/java.io.tmpdir/Jetty_localhost_44797_datanode____.qcs3no/webapp 2024-04-12T03:12:15,561 INFO [Listener at localhost/42429 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44797 2024-04-12T03:12:15,573 WARN [Listener at localhost/40485 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-12T03:12:15,590 WARN [Listener at localhost/40485 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-12T03:12:15,594 WARN [Listener at localhost/40485 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-12T03:12:15,596 INFO [Listener at localhost/40485 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-12T03:12:15,605 INFO [Listener at localhost/40485 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/java.io.tmpdir/Jetty_localhost_44217_datanode____.d3hdj5/webapp 2024-04-12T03:12:15,734 INFO [Listener at localhost/40485 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44217 2024-04-12T03:12:15,739 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-12T03:12:15,739 WARN [Thread-74 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-12T03:12:15,752 WARN [Listener at localhost/39223 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-12T03:12:15,869 WARN [Thread-99 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-12T03:12:16,013 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x329b0e136215794e: Processing first storage report for DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f from datanode fb3c79b6-2a41-404e-8c11-b21bb1709440 2024-04-12T03:12:16,015 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x329b0e136215794e: from storage DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f node DatanodeRegistration(127.0.0.1:43033, datanodeUuid=fb3c79b6-2a41-404e-8c11-b21bb1709440, infoPort=41077, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,015 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xaa37084020aca62d: Processing first storage report for DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74 from datanode a7fa59b7-b1cf-441c-8938-8fb930090e5a 2024-04-12T03:12:16,015 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xaa37084020aca62d: from storage DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74 node DatanodeRegistration(127.0.0.1:34801, datanodeUuid=a7fa59b7-b1cf-441c-8938-8fb930090e5a, infoPort=44069, infoSecurePort=0, ipcPort=40485, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,016 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xee59fd354da964c: Processing first storage report for DS-12d42e62-dc37-4557-b6e3-799e735bcdac from datanode f0646ff2-8772-444d-8c1a-f802bece4af2 2024-04-12T03:12:16,016 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xee59fd354da964c: from storage DS-12d42e62-dc37-4557-b6e3-799e735bcdac node DatanodeRegistration(127.0.0.1:41309, datanodeUuid=f0646ff2-8772-444d-8c1a-f802bece4af2, infoPort=39975, infoSecurePort=0, ipcPort=39223, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,016 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x329b0e136215794e: Processing first storage report for DS-ddc287d4-f162-4752-a80e-6716a7fb7150 from datanode fb3c79b6-2a41-404e-8c11-b21bb1709440 2024-04-12T03:12:16,016 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x329b0e136215794e: from storage DS-ddc287d4-f162-4752-a80e-6716a7fb7150 node DatanodeRegistration(127.0.0.1:43033, datanodeUuid=fb3c79b6-2a41-404e-8c11-b21bb1709440, infoPort=41077, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,016 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xaa37084020aca62d: Processing first storage report for DS-fb3bca85-b373-4a21-9d45-b7e630397558 from datanode a7fa59b7-b1cf-441c-8938-8fb930090e5a 2024-04-12T03:12:16,017 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xaa37084020aca62d: from storage DS-fb3bca85-b373-4a21-9d45-b7e630397558 node DatanodeRegistration(127.0.0.1:34801, datanodeUuid=a7fa59b7-b1cf-441c-8938-8fb930090e5a, infoPort=44069, infoSecurePort=0, ipcPort=40485, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,017 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xee59fd354da964c: Processing first storage report for DS-8694eee6-38f0-4332-ae47-81879818077c from datanode f0646ff2-8772-444d-8c1a-f802bece4af2 2024-04-12T03:12:16,017 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xee59fd354da964c: from storage DS-8694eee6-38f0-4332-ae47-81879818077c node DatanodeRegistration(127.0.0.1:41309, datanodeUuid=f0646ff2-8772-444d-8c1a-f802bece4af2, infoPort=39975, infoSecurePort=0, ipcPort=39223, storageInfo=lv=-57;cid=testClusterID;nsid=92476180;c=1712891533757), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-04-12T03:12:16,302 DEBUG [Listener at localhost/39223 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293 2024-04-12T03:12:16,328 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[0: nbRows=1,500, walEditKVs=1, isCompressionEnabled=false] Thread=141, OpenFileDescriptor=445, MaxFileDescriptor=60000, SystemLoadAverage=300, ProcessCount=171, AvailableMemoryMB=4085 2024-04-12T03:12:16,332 DEBUG [Listener at localhost/39223 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-04-12T03:12:17,002 INFO [Listener at localhost/39223 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-04-12T03:12:17,031 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:17,045 DEBUG [Listener at localhost/39223 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-04-12T03:12:17,107 DEBUG [Listener at localhost/39223 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-04-12T03:12:17,107 DEBUG [Listener at localhost/39223 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-04-12T03:12:17,144 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:17,200 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:17,298 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2024-04-12T03:12:17,298 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2024-04-12T03:12:17,300 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:54) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:183) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:123) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:694) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:917) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$8(AbstractFSWAL.java:949) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:949) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:583) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:524) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:157) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:59) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:325) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStreamTestBase.initWAL(WALEntryStreamTestBase.java:155) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestWALEntryStreamDifferentCounts.setUp(TestWALEntryStreamDifferentCounts.java:63) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:17,310 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244, exclude list is [], retry=0 2024-04-12T03:12:17,530 DEBUG [Listener at localhost/39223 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 5325 (auto-detected) 2024-04-12T03:12:17,537 DEBUG [Listener at localhost/39223 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:5a:ff:fe:dc:8e:c9 (auto-detected) 2024-04-12T03:12:17,593 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:301) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.6.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:17,605 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:17,613 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:301) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.6.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:17,654 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:17,654 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:17,798 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244 2024-04-12T03:12:17,799 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK]] 2024-04-12T03:12:19,442 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891539427, exclude list is [], retry=0 2024-04-12T03:12:19,450 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:19,454 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:19,455 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:19,468 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244 with entries=1500, filesize=184.53 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891539427 2024-04-12T03:12:19,471 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:19,471 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244 is not closed yet, will try archiving it next time 2024-04-12T03:12:19,474 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244, startPosition=0, beingWritten=true 2024-04-12T03:12:19,635 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891537244 2024-04-12T03:12:19,642 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891539427, startPosition=0, beingWritten=true 2024-04-12T03:12:19,743 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:12:19,744 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D:(num 1712891539427) 2024-04-12T03:12:19,758 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[0: nbRows=1,500, walEditKVs=1, isCompressionEnabled=false] Thread=148 (was 141) Potentially hanging thread: AsyncFSWAL-1-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-947737041_18 at /127.0.0.1:55344 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Parameter Sending Thread #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HBase-Metrics2-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-947737041_18 at /127.0.0.1:54018 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33707 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@94d732b java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=467 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=349 (was 300) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=3970 (was 4085) 2024-04-12T03:12:19,771 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[1: nbRows=1,500, walEditKVs=1, isCompressionEnabled=true] Thread=148, OpenFileDescriptor=467, MaxFileDescriptor=60000, SystemLoadAverage=349, ProcessCount=171, AvailableMemoryMB=3970 2024-04-12T03:12:19,772 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:19,774 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:19,779 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:19,795 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781, exclude list is [], retry=0 2024-04-12T03:12:19,802 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:19,804 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:19,804 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:19,811 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781 2024-04-12T03:12:19,811 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:20,768 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891540754, exclude list is [], retry=0 2024-04-12T03:12:20,775 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:20,776 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:20,777 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:20,781 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781 with entries=1500, filesize=89.08 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891540754 2024-04-12T03:12:20,782 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:20,782 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781 is not closed yet, will try archiving it next time 2024-04-12T03:12:20,784 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781, startPosition=0, beingWritten=true 2024-04-12T03:12:20,793 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:12:20,824 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891539781 2024-04-12T03:12:20,825 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891540754, startPosition=0, beingWritten=true 2024-04-12T03:12:20,835 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891540754: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:12:20,851 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:12:20,851 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D:(num 1712891540754) 2024-04-12T03:12:20,864 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[1: nbRows=1,500, walEditKVs=1, isCompressionEnabled=true] Thread=149 (was 148) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-947737041_18 at /127.0.0.1:54018 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: AsyncFSWAL-3-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=476 (was 467) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=349 (was 349), ProcessCount=171 (was 171), AvailableMemoryMB=3947 (was 3970) 2024-04-12T03:12:20,876 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[2: nbRows=1,500, walEditKVs=100, isCompressionEnabled=false] Thread=149, OpenFileDescriptor=476, MaxFileDescriptor=60000, SystemLoadAverage=349, ProcessCount=171, AvailableMemoryMB=3946 2024-04-12T03:12:20,877 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:20,878 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:20,883 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:20,902 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885, exclude list is [], retry=0 2024-04-12T03:12:20,909 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:20,910 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:20,911 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:20,914 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885 2024-04-12T03:12:20,915 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:21,947 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891541935, exclude list is [], retry=0 2024-04-12T03:12:21,954 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:21,955 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:21,956 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:21,960 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885 with entries=1500, filesize=8.11 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891541935 2024-04-12T03:12:21,960 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK]] 2024-04-12T03:12:21,960 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885 is not closed yet, will try archiving it next time 2024-04-12T03:12:21,963 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885, startPosition=0, beingWritten=true 2024-04-12T03:12:22,152 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891540885 2024-04-12T03:12:22,152 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891541935, startPosition=0, beingWritten=true 2024-04-12T03:12:22,173 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:12:22,173 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D:(num 1712891541935) 2024-04-12T03:12:22,185 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[2: nbRows=1,500, walEditKVs=100, isCompressionEnabled=false] Thread=148 (was 149), OpenFileDescriptor=479 (was 476) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=349 (was 349), ProcessCount=171 (was 171), AvailableMemoryMB=3849 (was 3946) 2024-04-12T03:12:22,197 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[3: nbRows=1,500, walEditKVs=100, isCompressionEnabled=true] Thread=148, OpenFileDescriptor=479, MaxFileDescriptor=60000, SystemLoadAverage=349, ProcessCount=171, AvailableMemoryMB=3849 2024-04-12T03:12:22,199 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:22,200 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:22,205 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:22,219 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207, exclude list is [], retry=0 2024-04-12T03:12:22,226 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:22,228 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:22,228 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:22,233 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207 2024-04-12T03:12:22,233 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:22,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:12:23,355 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891543343, exclude list is [], retry=0 2024-04-12T03:12:23,362 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:23,363 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:23,364 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:23,367 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207 with entries=1500, filesize=3.91 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891543343 2024-04-12T03:12:23,367 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK]] 2024-04-12T03:12:23,367 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207 is not closed yet, will try archiving it next time 2024-04-12T03:12:23,369 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207, startPosition=0, beingWritten=true 2024-04-12T03:12:23,383 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:12:24,039 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891542207 2024-04-12T03:12:24,039 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891543343, startPosition=0, beingWritten=true 2024-04-12T03:12:24,048 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891543343: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:12:24,060 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:12:24,060 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D:(num 1712891543343) 2024-04-12T03:12:24,072 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[3: nbRows=1,500, walEditKVs=100, isCompressionEnabled=true] Thread=150 (was 148) Potentially hanging thread: AsyncFSWAL-5-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-947737041_18 at /127.0.0.1:53810 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) - Thread LEAK? -, OpenFileDescriptor=482 (was 479) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=345 (was 349), ProcessCount=171 (was 171), AvailableMemoryMB=3761 (was 3849) 2024-04-12T03:12:24,083 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[4: nbRows=60,000, walEditKVs=1, isCompressionEnabled=false] Thread=150, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=345, ProcessCount=171, AvailableMemoryMB=3761 2024-04-12T03:12:24,084 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:24,085 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:24,089 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:24,101 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090, exclude list is [], retry=0 2024-04-12T03:12:24,108 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:24,109 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:24,110 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:24,113 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090 2024-04-12T03:12:24,113 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:27,701 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:12:47,668 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891567657, exclude list is [], retry=0 2024-04-12T03:12:47,675 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:47,675 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:47,676 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:47,680 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090 with entries=60000, filesize=7.25 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891567657 2024-04-12T03:12:47,680 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:47,681 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090 is not closed yet, will try archiving it next time 2024-04-12T03:12:47,682 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090, startPosition=0, beingWritten=true 2024-04-12T03:12:47,872 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891544090 2024-04-12T03:12:47,872 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1712891567657, startPosition=0, beingWritten=true 2024-04-12T03:12:47,890 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:12:47,890 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D:(num 1712891567657) 2024-04-12T03:12:47,902 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[4: nbRows=60,000, walEditKVs=1, isCompressionEnabled=false] Thread=149 (was 150), OpenFileDescriptor=479 (was 482), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=281 (was 345), ProcessCount=171 (was 171), AvailableMemoryMB=4871 (was 3761) - AvailableMemoryMB LEAK? - 2024-04-12T03:12:47,913 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[5: nbRows=60,000, walEditKVs=1, isCompressionEnabled=true] Thread=149, OpenFileDescriptor=479, MaxFileDescriptor=60000, SystemLoadAverage=281, ProcessCount=171, AvailableMemoryMB=4871 2024-04-12T03:12:47,914 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:12:47,915 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:12:47,919 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:12:47,932 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920, exclude list is [], retry=0 2024-04-12T03:12:47,938 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:12:47,939 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:12:47,940 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:12:47,943 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920 2024-04-12T03:12:47,943 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:12:53,430 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:13:12,084 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891592070, exclude list is [], retry=0 2024-04-12T03:13:12,093 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:13:12,094 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:13:12,095 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:13:12,099 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920 with entries=60000, filesize=3.43 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891592070 2024-04-12T03:13:12,099 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK]] 2024-04-12T03:13:12,099 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920 is not closed yet, will try archiving it next time 2024-04-12T03:13:12,101 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920, startPosition=0, beingWritten=true 2024-04-12T03:13:12,113 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:13:12,555 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891567920 2024-04-12T03:13:12,556 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891592070, startPosition=0, beingWritten=true 2024-04-12T03:13:12,564 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1712891592070: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:13:12,575 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:13:12,576 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D:(num 1712891592070) 2024-04-12T03:13:12,591 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[5: nbRows=60,000, walEditKVs=1, isCompressionEnabled=true] Thread=148 (was 149), OpenFileDescriptor=482 (was 479) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=353 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=4674 (was 4871) 2024-04-12T03:13:12,607 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[6: nbRows=60,000, walEditKVs=100, isCompressionEnabled=false] Thread=148, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=353, ProcessCount=171, AvailableMemoryMB=4673 2024-04-12T03:13:12,608 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:13:12,609 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:13:12,614 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:13:12,633 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615, exclude list is [], retry=0 2024-04-12T03:13:12,642 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:13:12,643 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:13:12,644 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:13:12,647 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615 2024-04-12T03:13:12,647 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK]] 2024-04-12T03:13:18,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:13:23,521 DEBUG [AsyncFSWAL-0-hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293 {}] wal.AsyncFSWAL(389): Requesting log roll because of file size threshold; length=134221818, logrollsize=134217728 2024-04-12T03:13:40,706 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891620695, exclude list is [], retry=0 2024-04-12T03:13:40,714 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:13:40,714 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:13:40,715 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:13:40,719 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615 with entries=60000, filesize=324.48 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891620695 2024-04-12T03:13:40,720 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:13:40,720 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615 is not closed yet, will try archiving it next time 2024-04-12T03:13:40,721 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615, startPosition=0, beingWritten=true 2024-04-12T03:13:45,882 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891592615 2024-04-12T03:13:45,883 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1712891620695, startPosition=0, beingWritten=true 2024-04-12T03:13:45,900 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:13:45,900 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D:(num 1712891620695) 2024-04-12T03:13:45,911 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[6: nbRows=60,000, walEditKVs=100, isCompressionEnabled=false] Thread=134 (was 148), OpenFileDescriptor=483 (was 482) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=355 (was 353) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=3325 (was 4673) 2024-04-12T03:13:45,922 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[7: nbRows=60,000, walEditKVs=100, isCompressionEnabled=true] Thread=134, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=355, ProcessCount=171, AvailableMemoryMB=3325 2024-04-12T03:13:45,923 INFO [Listener at localhost/39223 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-12T03:13:45,924 INFO [Listener at localhost/39223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-12T03:13:45,928 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true], archiveDir=hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs, maxLogs=32 2024-04-12T03:13:45,939 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928, exclude list is [], retry=0 2024-04-12T03:13:45,946 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:13:45,946 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:13:45,947 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:13:45,949 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928 2024-04-12T03:13:45,950 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:13:51,436 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:14:10,318 DEBUG [AsyncFSWAL-0-hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293 {}] wal.AsyncFSWAL(389): Requesting log roll because of file size threshold; length=134218993, logrollsize=134217728 2024-04-12T03:14:15,712 DEBUG [Listener at localhost/39223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891655700, exclude list is [], retry=0 2024-04-12T03:14:15,721 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK] 2024-04-12T03:14:15,722 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK] 2024-04-12T03:14:15,722 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK] 2024-04-12T03:14:15,726 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928 with entries=60000, filesize=156.41 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891655700 2024-04-12T03:14:15,726 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34801,DS-a94f4430-5bd2-4a8c-95b0-210ef39dbb74,DISK], DatanodeInfoWithStorage[127.0.0.1:43033,DS-5a5c05ea-1aab-467e-b06e-fb17a8a6117f,DISK], DatanodeInfoWithStorage[127.0.0.1:41309,DS-12d42e62-dc37-4557-b6e3-799e735bcdac,DISK]] 2024-04-12T03:14:15,727 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(749): hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928 is not closed yet, will try archiving it next time 2024-04-12T03:14:15,727 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928, startPosition=0, beingWritten=true 2024-04-12T03:14:15,738 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:14:38,988 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891625928 2024-04-12T03:14:38,989 DEBUG [Listener at localhost/39223 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891655700, startPosition=0, beingWritten=true 2024-04-12T03:14:38,999 DEBUG [Listener at localhost/39223 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33707/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1712891655700: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-12T03:14:39,011 DEBUG [Listener at localhost/39223 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/oldWALs 2024-04-12T03:14:39,011 INFO [Listener at localhost/39223 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D:(num 1712891655700) 2024-04-12T03:14:39,022 INFO [Listener at localhost/39223 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[7: nbRows=60,000, walEditKVs=100, isCompressionEnabled=true] Thread=136 (was 134) - Thread LEAK? -, OpenFileDescriptor=488 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=235 (was 355), ProcessCount=168 (was 171), AvailableMemoryMB=2695 (was 3325) 2024-04-12T03:14:39,023 INFO [Listener at localhost/39223 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-04-12T03:14:39,023 WARN [Listener at localhost/39223 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-12T03:14:39,028 INFO [Listener at localhost/39223 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-12T03:14:39,131 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-12T03:14:39,132 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1737496199-172.31.2.20-1712891533757 (Datanode Uuid f0646ff2-8772-444d-8c1a-f802bece4af2) service to localhost/127.0.0.1:33707 2024-04-12T03:14:39,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data5/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data6/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,137 WARN [Listener at localhost/39223 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-12T03:14:39,141 INFO [Listener at localhost/39223 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-12T03:14:39,244 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-12T03:14:39,245 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1737496199-172.31.2.20-1712891533757 (Datanode Uuid a7fa59b7-b1cf-441c-8938-8fb930090e5a) service to localhost/127.0.0.1:33707 2024-04-12T03:14:39,249 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data3/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,249 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data4/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,250 WARN [Listener at localhost/39223 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-12T03:14:39,254 INFO [Listener at localhost/39223 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-12T03:14:39,357 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-12T03:14:39,357 WARN [BP-1737496199-172.31.2.20-1712891533757 heartbeating to localhost/127.0.0.1:33707 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1737496199-172.31.2.20-1712891533757 (Datanode Uuid fb3c79b6-2a41-404e-8c11-b21bb1709440) service to localhost/127.0.0.1:33707 2024-04-12T03:14:39,368 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data1/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,369 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2b31678-9769-30e8-a1a2-02de6d86f293/cluster_426f4e48-8d10-cc29-03af-2afffdf3de33/dfs/data/data2/current/BP-1737496199-172.31.2.20-1712891533757 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-12T03:14:39,397 INFO [Listener at localhost/39223 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-12T03:14:39,760 INFO [Listener at localhost/39223 {}] hbase.HBaseTestingUtility(1347): Minicluster is down