2024-04-14 11:11:59,384 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 2024-04-14 11:11:59,397 main DEBUG Took 0.011242 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-04-14 11:11:59,397 main DEBUG PluginManager 'Core' found 129 plugins 2024-04-14 11:11:59,398 main DEBUG PluginManager 'Level' found 0 plugins 2024-04-14 11:11:59,399 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-04-14 11:11:59,400 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,409 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-04-14 11:11:59,421 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,424 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,425 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,426 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,427 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,427 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,428 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,428 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,429 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,430 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,430 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,431 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:11:59,431 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,431 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-04-14 11:11:59,433 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:11:59,435 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-04-14 11:11:59,437 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-04-14 11:11:59,438 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-04-14 11:11:59,439 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-04-14 11:11:59,439 main DEBUG PluginManager 'Converter' found 47 plugins 2024-04-14 11:11:59,450 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-04-14 11:11:59,453 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-04-14 11:11:59,457 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-04-14 11:11:59,458 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-04-14 11:11:59,458 main DEBUG createAppenders(={Console}) 2024-04-14 11:11:59,459 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 initialized 2024-04-14 11:11:59,459 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 2024-04-14 11:11:59,459 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15ca7889 OK. 2024-04-14 11:11:59,460 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-04-14 11:11:59,460 main DEBUG OutputStream closed 2024-04-14 11:11:59,460 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-04-14 11:11:59,461 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-04-14 11:11:59,461 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55141def OK 2024-04-14 11:11:59,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-04-14 11:11:59,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-04-14 11:11:59,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-04-14 11:11:59,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-04-14 11:11:59,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-04-14 11:11:59,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-04-14 11:11:59,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-04-14 11:11:59,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-04-14 11:11:59,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-04-14 11:11:59,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-04-14 11:11:59,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-04-14 11:11:59,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-04-14 11:11:59,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-04-14 11:11:59,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-04-14 11:11:59,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-04-14 11:11:59,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-04-14 11:11:59,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-04-14 11:11:59,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-04-14 11:11:59,518 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14 11:11:59,519 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@73e9cf30) with optional ClassLoader: null 2024-04-14 11:11:59,519 main DEBUG Shutdown hook enabled. Registering a new one. 2024-04-14 11:11:59,520 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@73e9cf30] started OK. 2024-04-14T11:11:59,779 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa 2024-04-14 11:11:59,783 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-04-14 11:11:59,783 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14T11:11:59,829 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL timeout: 104 mins 2024-04-14T11:11:59,843 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc, deleteOnExit=true 2024-04-14T11:11:59,844 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/test.cache.data in system properties and HBase conf 2024-04-14T11:11:59,844 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/hadoop.tmp.dir in system properties and HBase conf 2024-04-14T11:11:59,845 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/hadoop.log.dir in system properties and HBase conf 2024-04-14T11:11:59,845 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-04-14T11:11:59,846 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-04-14T11:11:59,846 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-04-14T11:11:59,952 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-04-14T11:12:00,382 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:00,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-04-14T11:12:00,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-04-14T11:12:00,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-04-14T11:12:00,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-14T11:12:00,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-04-14T11:12:00,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-04-14T11:12:00,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-14T11:12:00,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-14T11:12:00,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-04-14T11:12:00,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/nfs.dump.dir in system properties and HBase conf 2024-04-14T11:12:00,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/java.io.tmpdir in system properties and HBase conf 2024-04-14T11:12:00,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-14T11:12:00,391 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-04-14T11:12:00,391 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2024-04-14T11:12:00,873 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-14T11:12:00,878 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-14T11:12:01,185 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-04-14T11:12:01,308 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@624f77e2 via org.mortbay.log.Slf4jLog 2024-04-14T11:12:01,324 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:12:01,363 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:12:01,399 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/java.io.tmpdir/Jetty_localhost_40077_hdfs____pnioqm/webapp 2024-04-14T11:12:01,534 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40077 2024-04-14T11:12:01,558 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-14T11:12:01,559 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-14T11:12:02,134 WARN [Listener at localhost/33437 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:12:02,204 WARN [Listener at localhost/33437 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-14T11:12:02,222 WARN [Listener at localhost/33437 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:12:02,229 INFO [Listener at localhost/33437 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:12:02,234 INFO [Listener at localhost/33437 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/java.io.tmpdir/Jetty_localhost_40903_datanode____9vngmf/webapp 2024-04-14T11:12:02,333 INFO [Listener at localhost/33437 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40903 2024-04-14T11:12:02,564 WARN [Listener at localhost/44733 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:12:02,580 WARN [Listener at localhost/44733 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-14T11:12:02,585 WARN [Listener at localhost/44733 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:12:02,587 INFO [Listener at localhost/44733 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:12:02,593 INFO [Listener at localhost/44733 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/java.io.tmpdir/Jetty_localhost_42629_datanode____ef6ply/webapp 2024-04-14T11:12:02,696 INFO [Listener at localhost/44733 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42629 2024-04-14T11:12:02,705 WARN [Listener at localhost/43849 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:12:02,722 WARN [Listener at localhost/43849 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-14T11:12:02,725 WARN [Listener at localhost/43849 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:12:02,728 INFO [Listener at localhost/43849 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:12:02,738 INFO [Listener at localhost/43849 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/java.io.tmpdir/Jetty_localhost_40155_datanode____.rztkw4/webapp 2024-04-14T11:12:02,851 INFO [Listener at localhost/43849 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40155 2024-04-14T11:12:02,872 WARN [Listener at localhost/45645 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:12:02,882 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-14T11:12:02,882 WARN [Thread-74 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-14T11:12:02,997 WARN [Thread-99 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-14T11:12:03,179 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf8d6079f1dbf6ca4: Processing first storage report for DS-88dac98d-ba63-4976-852f-fda5c57fdaf9 from datanode c39ce524-441c-49d7-ac22-7c2be2f9abc1 2024-04-14T11:12:03,181 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8d6079f1dbf6ca4: from storage DS-88dac98d-ba63-4976-852f-fda5c57fdaf9 node DatanodeRegistration(127.0.0.1:40931, datanodeUuid=c39ce524-441c-49d7-ac22-7c2be2f9abc1, infoPort=46815, infoSecurePort=0, ipcPort=45645, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x67dd498fd9c5e625: Processing first storage report for DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf from datanode 2d0ba493-cbfb-4a37-8d0d-daa849998e76 2024-04-14T11:12:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x67dd498fd9c5e625: from storage DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf node DatanodeRegistration(127.0.0.1:40585, datanodeUuid=2d0ba493-cbfb-4a37-8d0d-daa849998e76, infoPort=43385, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x55baa6dc1936e50f: Processing first storage report for DS-a3a281e1-088a-445a-8244-e718c8a8506f from datanode b6aff713-600c-4b97-a708-0755e8a1c14b 2024-04-14T11:12:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x55baa6dc1936e50f: from storage DS-a3a281e1-088a-445a-8244-e718c8a8506f node DatanodeRegistration(127.0.0.1:36283, datanodeUuid=b6aff713-600c-4b97-a708-0755e8a1c14b, infoPort=36409, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf8d6079f1dbf6ca4: Processing first storage report for DS-e3d5410d-a4f4-4d80-b6ad-74544d00dfb0 from datanode c39ce524-441c-49d7-ac22-7c2be2f9abc1 2024-04-14T11:12:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8d6079f1dbf6ca4: from storage DS-e3d5410d-a4f4-4d80-b6ad-74544d00dfb0 node DatanodeRegistration(127.0.0.1:40931, datanodeUuid=c39ce524-441c-49d7-ac22-7c2be2f9abc1, infoPort=46815, infoSecurePort=0, ipcPort=45645, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x67dd498fd9c5e625: Processing first storage report for DS-dbae190b-2f9d-4812-9949-16fd95d286ed from datanode 2d0ba493-cbfb-4a37-8d0d-daa849998e76 2024-04-14T11:12:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x67dd498fd9c5e625: from storage DS-dbae190b-2f9d-4812-9949-16fd95d286ed node DatanodeRegistration(127.0.0.1:40585, datanodeUuid=2d0ba493-cbfb-4a37-8d0d-daa849998e76, infoPort=43385, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x55baa6dc1936e50f: Processing first storage report for DS-390959c5-e0f8-4b22-a255-370b7f7f3fb2 from datanode b6aff713-600c-4b97-a708-0755e8a1c14b 2024-04-14T11:12:03,184 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x55baa6dc1936e50f: from storage DS-390959c5-e0f8-4b22-a255-370b7f7f3fb2 node DatanodeRegistration(127.0.0.1:36283, datanodeUuid=b6aff713-600c-4b97-a708-0755e8a1c14b, infoPort=36409, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=2091467869;c=1713093120956), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:12:03,371 DEBUG [Listener at localhost/45645 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa 2024-04-14T11:12:03,393 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[0: nbRows=1,500, walEditKVs=1, isCompressionEnabled=false] Thread=141, OpenFileDescriptor=445, MaxFileDescriptor=60000, SystemLoadAverage=150, ProcessCount=170, AvailableMemoryMB=3376 2024-04-14T11:12:03,397 DEBUG [Listener at localhost/45645 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-04-14T11:12:04,119 INFO [Listener at localhost/45645 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-04-14T11:12:04,150 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:04,165 DEBUG [Listener at localhost/45645 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-04-14T11:12:04,231 DEBUG [Listener at localhost/45645 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-04-14T11:12:04,232 DEBUG [Listener at localhost/45645 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-04-14T11:12:04,272 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:04,340 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:04,442 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2024-04-14T11:12:04,442 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2024-04-14T11:12:04,444 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:54) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:183) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:123) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:694) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:917) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$8(AbstractFSWAL.java:949) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:949) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:583) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:524) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:157) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:59) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:325) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStreamTestBase.initWAL(WALEntryStreamTestBase.java:155) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestWALEntryStreamDifferentCounts.setUp(TestWALEntryStreamDifferentCounts.java:63) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:04,455 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383, exclude list is [], retry=0 2024-04-14T11:12:04,686 DEBUG [Listener at localhost/45645 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 27875 (auto-detected) 2024-04-14T11:12:04,693 DEBUG [Listener at localhost/45645 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:5a:ff:fe:dc:8e:c9 (auto-detected) 2024-04-14T11:12:04,749 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:301) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.6.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:04,759 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:04,767 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:301) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.6.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:04,812 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:04,813 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:04,961 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383 2024-04-14T11:12:04,962 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK]] 2024-04-14T11:12:06,317 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093126299, exclude list is [], retry=0 2024-04-14T11:12:06,327 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:06,328 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:06,329 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:06,336 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383 with entries=1500, filesize=184.53 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093126299 2024-04-14T11:12:06,338 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:12:06,338 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383 is not closed yet, will try archiving it next time 2024-04-14T11:12:06,343 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383, startPosition=0, beingWritten=true 2024-04-14T11:12:06,491 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093124383 2024-04-14T11:12:06,502 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_0_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093126299, startPosition=0, beingWritten=true 2024-04-14T11:12:06,616 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:06,617 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_0_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D:(num 1713093126299) 2024-04-14T11:12:06,632 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[0: nbRows=1,500, walEditKVs=1, isCompressionEnabled=false] Thread=147 (was 141) Potentially hanging thread: AsyncFSWAL-1-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HBase-Metrics2-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:49414 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:34050 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33437 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@47c2cd6e java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=467 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=154 (was 150) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=3295 (was 3376) 2024-04-14T11:12:06,646 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[1: nbRows=1,500, walEditKVs=1, isCompressionEnabled=true] Thread=147, OpenFileDescriptor=467, MaxFileDescriptor=60000, SystemLoadAverage=154, ProcessCount=170, AvailableMemoryMB=3294 2024-04-14T11:12:06,647 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:06,648 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:06,654 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:06,670 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655, exclude list is [], retry=0 2024-04-14T11:12:06,677 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:06,679 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:06,680 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:06,687 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655 2024-04-14T11:12:06,688 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK]] 2024-04-14T11:12:07,584 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093127567, exclude list is [], retry=0 2024-04-14T11:12:07,594 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:07,595 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:07,596 DEBUG [AsyncFSWAL-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:07,608 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655 with entries=1500, filesize=88.82 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093127567 2024-04-14T11:12:07,608 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:12:07,608 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655 is not closed yet, will try archiving it next time 2024-04-14T11:12:07,610 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655, startPosition=0, beingWritten=true 2024-04-14T11:12:07,630 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:07,663 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093126655 2024-04-14T11:12:07,663 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093127567, startPosition=0, beingWritten=true 2024-04-14T11:12:07,674 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_1_ nbRows=1,500, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093127567: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:07,691 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:07,691 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_1_+nbRows%3D1%2C500%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D:(num 1713093127567) 2024-04-14T11:12:07,708 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[1: nbRows=1,500, walEditKVs=1, isCompressionEnabled=true] Thread=149 (was 147) Potentially hanging thread: IPC Parameter Sending Thread #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:34050 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:46600 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: AsyncFSWAL-3-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=481 (was 467) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=154 (was 154), ProcessCount=170 (was 170), AvailableMemoryMB=3283 (was 3294) 2024-04-14T11:12:07,722 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[2: nbRows=1,500, walEditKVs=100, isCompressionEnabled=false] Thread=149, OpenFileDescriptor=481, MaxFileDescriptor=60000, SystemLoadAverage=154, ProcessCount=170, AvailableMemoryMB=3282 2024-04-14T11:12:07,723 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:07,724 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:07,730 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:07,743 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731, exclude list is [], retry=0 2024-04-14T11:12:07,751 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:07,754 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:07,755 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:07,759 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731 2024-04-14T11:12:07,761 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK]] 2024-04-14T11:12:08,626 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093128615, exclude list is [], retry=0 2024-04-14T11:12:08,633 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:08,634 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:08,635 DEBUG [AsyncFSWAL-4-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:08,639 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731 with entries=1500, filesize=8.11 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093128615 2024-04-14T11:12:08,639 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK]] 2024-04-14T11:12:08,639 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731 is not closed yet, will try archiving it next time 2024-04-14T11:12:08,640 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731, startPosition=0, beingWritten=true 2024-04-14T11:12:08,835 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093127731 2024-04-14T11:12:08,835 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_2_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093128615, startPosition=0, beingWritten=true 2024-04-14T11:12:08,857 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:08,857 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_2_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D:(num 1713093128615) 2024-04-14T11:12:08,870 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[2: nbRows=1,500, walEditKVs=100, isCompressionEnabled=false] Thread=150 (was 149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:49474 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:34050 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: AsyncFSWAL-4-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=484 (was 481) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=182 (was 154) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=3224 (was 3282) 2024-04-14T11:12:08,882 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[3: nbRows=1,500, walEditKVs=100, isCompressionEnabled=true] Thread=150, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=182, ProcessCount=170, AvailableMemoryMB=3224 2024-04-14T11:12:08,884 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:08,885 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:08,890 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:08,907 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893, exclude list is [], retry=0 2024-04-14T11:12:08,915 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:08,916 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:08,917 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:08,923 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893 2024-04-14T11:12:08,923 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:12:09,645 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:12:09,916 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093129905, exclude list is [], retry=0 2024-04-14T11:12:09,924 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:09,925 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:09,925 DEBUG [AsyncFSWAL-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:09,960 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893 with entries=1500, filesize=3.91 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093129905 2024-04-14T11:12:09,961 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK]] 2024-04-14T11:12:09,961 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893 is not closed yet, will try archiving it next time 2024-04-14T11:12:09,965 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893, startPosition=0, beingWritten=true 2024-04-14T11:12:09,975 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:10,577 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093128893 2024-04-14T11:12:10,578 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093129905, startPosition=0, beingWritten=true 2024-04-14T11:12:10,587 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093129905: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:10,594 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(660): complete file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_3_ nbRows=1,500, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093129905 not finished, retry = 0 2024-04-14T11:12:10,706 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:10,706 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_3_+nbRows%3D1%2C500%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D:(num 1713093129905) 2024-04-14T11:12:10,718 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[3: nbRows=1,500, walEditKVs=100, isCompressionEnabled=true] Thread=151 (was 150) Potentially hanging thread: AsyncFSWAL-5-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:34098 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1781070448_18 at /127.0.0.1:49474 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) - Thread LEAK? -, OpenFileDescriptor=487 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=182 (was 182), ProcessCount=170 (was 170), AvailableMemoryMB=3075 (was 3224) 2024-04-14T11:12:10,729 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[4: nbRows=60,000, walEditKVs=1, isCompressionEnabled=false] Thread=151, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=182, ProcessCount=170, AvailableMemoryMB=3074 2024-04-14T11:12:10,731 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:10,731 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:10,736 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:10,749 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737, exclude list is [], retry=0 2024-04-14T11:12:10,756 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:10,758 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:10,759 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:10,762 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737 2024-04-14T11:12:10,765 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK]] 2024-04-14T11:12:16,233 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:12:33,962 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093153950, exclude list is [], retry=0 2024-04-14T11:12:33,970 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:33,971 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:33,971 DEBUG [AsyncFSWAL-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:33,976 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737 with entries=60000, filesize=7.25 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093153950 2024-04-14T11:12:33,977 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK]] 2024-04-14T11:12:33,978 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737 is not closed yet, will try archiving it next time 2024-04-14T11:12:33,979 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737, startPosition=0, beingWritten=true 2024-04-14T11:12:34,192 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093130737 2024-04-14T11:12:34,192 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_4_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=false]/testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D.1713093153950, startPosition=0, beingWritten=true 2024-04-14T11:12:34,213 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:34,213 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_4_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dfalse%5D:(num 1713093153950) 2024-04-14T11:12:34,226 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[4: nbRows=60,000, walEditKVs=1, isCompressionEnabled=false] Thread=148 (was 151), OpenFileDescriptor=479 (was 487), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=223 (was 182) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=2979 (was 3074) 2024-04-14T11:12:34,239 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[5: nbRows=60,000, walEditKVs=1, isCompressionEnabled=true] Thread=148, OpenFileDescriptor=479, MaxFileDescriptor=60000, SystemLoadAverage=223, ProcessCount=170, AvailableMemoryMB=2978 2024-04-14T11:12:34,240 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:34,241 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:34,246 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:34,260 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247, exclude list is [], retry=0 2024-04-14T11:12:34,267 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:34,268 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:34,268 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:34,272 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247 2024-04-14T11:12:34,272 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:12:39,756 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:12:57,807 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093177796, exclude list is [], retry=0 2024-04-14T11:12:57,816 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:57,817 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:57,817 DEBUG [AsyncFSWAL-7-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:57,821 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247 with entries=60000, filesize=3.42 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093177796 2024-04-14T11:12:57,822 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK]] 2024-04-14T11:12:57,822 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247 is not closed yet, will try archiving it next time 2024-04-14T11:12:57,823 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247, startPosition=0, beingWritten=true 2024-04-14T11:12:57,835 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:58,233 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093154247 2024-04-14T11:12:58,233 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093177796, startPosition=0, beingWritten=true 2024-04-14T11:12:58,242 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_5_ nbRows=60,000, walEditKVs=1, isCompressionEnabled=true]/testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D.1713093177796: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:12:58,253 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:12:58,253 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_5_+nbRows%3D60%2C000%2C+walEditKVs%3D1%2C+isCompressionEnabled%3Dtrue%5D:(num 1713093177796) 2024-04-14T11:12:58,265 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[5: nbRows=60,000, walEditKVs=1, isCompressionEnabled=true] Thread=146 (was 148), OpenFileDescriptor=480 (was 479) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=229 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=3176 (was 2978) - AvailableMemoryMB LEAK? - 2024-04-14T11:12:58,277 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[6: nbRows=60,000, walEditKVs=100, isCompressionEnabled=false] Thread=146, OpenFileDescriptor=480, MaxFileDescriptor=60000, SystemLoadAverage=229, ProcessCount=170, AvailableMemoryMB=3176 2024-04-14T11:12:58,278 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:12:58,279 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:12:58,283 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:12:58,295 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284, exclude list is [], retry=0 2024-04-14T11:12:58,302 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:12:58,303 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:12:58,304 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:12:58,306 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284 2024-04-14T11:12:58,307 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:13:03,794 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:13:08,466 DEBUG [AsyncFSWAL-0-hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa {}] wal.AsyncFSWAL(389): Requesting log roll because of file size threshold; length=134221818, logrollsize=134217728 2024-04-14T11:13:25,515 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093205503, exclude list is [], retry=0 2024-04-14T11:13:25,524 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:13:25,524 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:13:25,525 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:13:25,529 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284 with entries=60000, filesize=324.48 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093205503 2024-04-14T11:13:25,531 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK]] 2024-04-14T11:13:25,532 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284 is not closed yet, will try archiving it next time 2024-04-14T11:13:25,532 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284, startPosition=0, beingWritten=true 2024-04-14T11:13:30,955 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093178284 2024-04-14T11:13:30,955 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_6_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=false]/testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D.1713093205503, startPosition=0, beingWritten=true 2024-04-14T11:13:30,974 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:13:30,974 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_6_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dfalse%5D:(num 1713093205503) 2024-04-14T11:13:30,986 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[6: nbRows=60,000, walEditKVs=100, isCompressionEnabled=false] Thread=134 (was 146), OpenFileDescriptor=483 (was 480) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=256 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=1828 (was 3176) 2024-04-14T11:13:30,997 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(147): before: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[7: nbRows=60,000, walEditKVs=100, isCompressionEnabled=true] Thread=134, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=256, ProcessCount=170, AvailableMemoryMB=1828 2024-04-14T11:13:30,998 INFO [Listener at localhost/45645 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:13:30,999 INFO [Listener at localhost/45645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:13:31,003 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true], archiveDir=hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs, maxLogs=32 2024-04-14T11:13:31,016 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004, exclude list is [], retry=0 2024-04-14T11:13:31,022 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:13:31,024 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:13:31,024 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:13:31,027 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(839): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004 2024-04-14T11:13:31,027 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK]] 2024-04-14T11:13:36,512 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:13:58,504 DEBUG [AsyncFSWAL-0-hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa {}] wal.AsyncFSWAL(389): Requesting log roll because of file size threshold; length=134218259, logrollsize=134217728 2024-04-14T11:14:04,579 DEBUG [Listener at localhost/45645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093244566, exclude list is [], retry=0 2024-04-14T11:14:04,588 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK] 2024-04-14T11:14:04,589 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK] 2024-04-14T11:14:04,590 DEBUG [AsyncFSWAL-9-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK] 2024-04-14T11:14:04,594 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(835): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004 with entries=60000, filesize=156.43 MB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093244566 2024-04-14T11:14:04,595 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36283,DS-a3a281e1-088a-445a-8244-e718c8a8506f,DISK], DatanodeInfoWithStorage[127.0.0.1:40931,DS-88dac98d-ba63-4976-852f-fda5c57fdaf9,DISK], DatanodeInfoWithStorage[127.0.0.1:40585,DS-02e19ddd-4da5-438c-9678-c5adcd29a4bf,DISK]] 2024-04-14T11:14:04,595 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(749): hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004 is not closed yet, will try archiving it next time 2024-04-14T11:14:04,595 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004, startPosition=0, beingWritten=true 2024-04-14T11:14:04,606 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:14:28,828 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(457): EOF, closing hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093211004 2024-04-14T11:14:28,828 DEBUG [Listener at localhost/45645 {}] regionserver.WALEntryStream(255): Creating new reader hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093244566, startPosition=0, beingWritten=true 2024-04-14T11:14:28,844 DEBUG [Listener at localhost/45645 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33437/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/WALs/testDifferentCounts_7_ nbRows=60,000, walEditKVs=100, isCompressionEnabled=true]/testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D.1713093244566: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-04-14T11:14:28,859 DEBUG [Listener at localhost/45645 {}] wal.AbstractFSWAL(1068): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/oldWALs 2024-04-14T11:14:28,859 INFO [Listener at localhost/45645 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL testDifferentCounts_7_+nbRows%3D60%2C000%2C+walEditKVs%3D100%2C+isCompressionEnabled%3Dtrue%5D:(num 1713093244566) 2024-04-14T11:14:28,872 INFO [Listener at localhost/45645 {}] hbase.ResourceChecker(175): after: replication.regionserver.TestWALEntryStreamDifferentCountsAsyncFSWAL#testDifferentCounts[7: nbRows=60,000, walEditKVs=100, isCompressionEnabled=true] Thread=136 (was 134) - Thread LEAK? -, OpenFileDescriptor=488 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=268 (was 256) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 170), AvailableMemoryMB=864 (was 1828) 2024-04-14T11:14:28,872 INFO [Listener at localhost/45645 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-04-14T11:14:28,872 WARN [Listener at localhost/45645 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-14T11:14:28,878 INFO [Listener at localhost/45645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:14:28,982 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-14T11:14:28,982 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-41115233-172.31.2.20-1713093120956 (Datanode Uuid c39ce524-441c-49d7-ac22-7c2be2f9abc1) service to localhost/127.0.0.1:33437 2024-04-14T11:14:28,984 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data5/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:28,984 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data6/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:28,988 WARN [Listener at localhost/45645 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-14T11:14:28,990 INFO [Listener at localhost/45645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:14:29,093 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-14T11:14:29,093 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-41115233-172.31.2.20-1713093120956 (Datanode Uuid b6aff713-600c-4b97-a708-0755e8a1c14b) service to localhost/127.0.0.1:33437 2024-04-14T11:14:29,094 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data3/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:29,095 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data4/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:29,096 WARN [Listener at localhost/45645 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-14T11:14:29,098 INFO [Listener at localhost/45645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:14:29,200 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-14T11:14:29,200 WARN [BP-41115233-172.31.2.20-1713093120956 heartbeating to localhost/127.0.0.1:33437 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-41115233-172.31.2.20-1713093120956 (Datanode Uuid 2d0ba493-cbfb-4a37-8d0d-daa849998e76) service to localhost/127.0.0.1:33437 2024-04-14T11:14:29,211 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data1/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:29,212 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/45228d52-f9dc-8aba-a8f9-d574a12215fa/cluster_26449347-4266-cd88-8295-8d8aded66acc/dfs/data/data2/current/BP-41115233-172.31.2.20-1713093120956 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:14:29,238 INFO [Listener at localhost/45645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:14:29,605 INFO [Listener at localhost/45645 {}] hbase.HBaseTestingUtility(1347): Minicluster is down